diff --git a/nitter.example.conf b/nitter.example.conf
index f0b4214..100d875 100644
--- a/nitter.example.conf
+++ b/nitter.example.conf
@@ -26,12 +26,7 @@ enableRSS = true # set this to false to disable RSS feeds
enableDebug = false # enable request logs and debug endpoints (/.accounts)
proxy = "" # http/https url, SOCKS proxies are not supported
proxyAuth = ""
-tokenCount = 10
-# minimum amount of usable tokens. tokens are used to authorize API requests,
-# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
-# the limits reset every 15 minutes, and the pool is filled up so there's
-# always at least `tokenCount` usable tokens. only increase this if you receive
-# major bursts all the time and don't have a rate limiting setup via e.g. nginx
+disableTid = false # enable this if cookie-based auth is failing
# Change default preferences here, see src/prefs_impl.nim for a complete list
[Preferences]
diff --git a/src/api.nim b/src/api.nim
index 52ef057..5e837c4 100644
--- a/src/api.nim
+++ b/src/api.nim
@@ -1,78 +1,114 @@
# SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
+import asyncdispatch, httpclient, strutils, sequtils, sugar
import packedjson
import types, query, formatters, consts, apiutils, parser
import experimental/parser as newParser
+# Helper to generate params object for GraphQL requests
+proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
+ result.add ("variables", variables)
+ result.add ("features", gqlFeatures)
+ if fieldToggles.len > 0:
+ result.add ("fieldToggles", fieldToggles)
+
+proc apiUrl*(endpoint, variables: string; fieldToggles = ""): ApiUrl =
+ return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
+
+proc apiReq*(endpoint, variables: string; fieldToggles = ""): ApiReq =
+ let url = apiUrl(endpoint, variables, fieldToggles)
+ return ApiReq(cookie: url, oauth: url)
+
+proc mediaUrl*(id: string; cursor: string): ApiReq =
+ result = ApiReq(
+ cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor], """{"withArticlePlainText":false}"""),
+ oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor])
+ )
+
+proc userTweetsUrl*(id: string; cursor: string): ApiReq =
+ result = ApiReq(
+ cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+ oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor])
+ )
+
+proc userTweetsAndRepliesUrl*(id: string; cursor: string): ApiReq =
+ let cookieVars = userTweetsAndRepliesVars % [id, cursor]
+ result = ApiReq(
+ cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
+ oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor])
+ )
+
+proc tweetDetailUrl*(id: string; cursor: string): ApiReq =
+ result = ApiReq(
+ cookie: apiUrl(graphTweetDetail, tweetDetailVars % [id, cursor], tweetDetailFieldToggles),
+ oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+ )
+
+proc userUrl*(username: string): ApiReq =
+ let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
+ result = ApiReq(
+ cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
+ oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
+ )
+
proc getGraphUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
- let
- headers = newHttpHeaders()
+ let headers = newHttpHeaders()
headers.add("Referer", """https://x.com/$1""" % username)
- let
- variables = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
- fieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
- params = {"variables": variables, "features": userFeatures, "fieldToggles": fieldToggles}
- js = await fetch(graphUser ? params, Api.userScreenName, headers)
+ let js = await fetchRaw(userUrl(username), headers)
result = parseGraphUser(js)
proc getGraphUserById*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return
- let
- headers = newHttpHeaders()
+ let headers = newHttpHeaders()
headers.add("Referer", """https://x.com/i/user/$1""" % id)
let
- variables = """{"userId":"$1"}""" % id
- params = {"variables": variables, "features": userFeatures}
- js = await fetch(graphUserById ? params, Api.userRestId, headers)
+ url = apiReq(graphUserById, """{"userId":"$1"}""" % id)
+ js = await fetchRaw(url, headers)
result = parseGraphUser(js)
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
if id.len == 0: return
- let endpoint = case kind
- of TimelineKind.tweets: ""
- of TimelineKind.replies: "/with_replies"
- of TimelineKind.media: "/media"
let
+ endpoint = case kind
+ of TimelineKind.tweets: ""
+ of TimelineKind.replies: "/with_replies"
+ of TimelineKind.media: "/media"
headers = newHttpHeaders()
headers.add("Referer", """https://x.com/$1$2""" % [id, endpoint])
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = if kind == TimelineKind.media: userMediaVariables % [id, cursor] else: userTweetsVariables % [id, cursor]
- fieldToggles = """{"withArticlePlainText":true}"""
- params = {"variables": variables, "features": gqlFeatures, "fieldToggles": fieldToggles}
- (url, apiId) = case kind
- of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
- of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
- of TimelineKind.media: (graphUserMedia, Api.userMedia)
- js = await fetch(url ? params, apiId, headers)
+ url = case kind
+ of TimelineKind.tweets: userTweetsUrl(id, cursor)
+ of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
+ of TimelineKind.media: mediaUrl(id, cursor)
+ js = await fetch(url, headers)
result = parseGraphTimeline(js, if kind == TimelineKind.media: "" else: "user", after)
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = listTweetsVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- js = await fetch(graphListTweets ? params, Api.listTweets)
+ url = apiReq(graphListTweets, restIdVars % [id, cursor])
+ js = await fetch(url)
result = parseGraphTimeline(js, "list", after).tweets
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
variables = %*{"screenName": name, "listSlug": list}
- params = {"variables": $variables, "features": gqlFeatures}
- result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
+ url = apiReq(graphListBySlug, $variables)
+ js = await fetch(url)
+ result = parseGraphList(js)
proc getGraphList*(id: string): Future[List] {.async.} =
- let
- variables = """{"listId":"$1"}""" % id
- params = {"variables": variables, "features": gqlFeatures}
- result = parseGraphList(await fetch(graphListById ? params, Api.list))
+ let
+ url = apiReq(graphListById, """{"listId": "$1"}""" % id)
+ js = await fetch(url)
+ result = parseGraphList(js)
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
if list.id.len == 0: return
@@ -86,85 +122,45 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
}
if after.len > 0:
variables["cursor"] = % after
- let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
- result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
-
-proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} =
- if id.len == 0: return
- var
- variables = %*{
- "userId": id,
- "includePromotedContent":false,
- "withClientEventToken":false,
- "withBirdwatchNotes":true,
- "withVoice":true,
- "withV2Timeline":false
- }
- if after.len > 0:
- variables["cursor"] = % after
- let
- url = consts.favorites ? {"variables": $variables, "features": gqlFeatures}
- result = parseGraphTimeline(await fetch(url, Api.favorites), after)
+ let
+ url = apiReq(graphListMembers, $variables)
+ js = await fetchRaw(url)
+ result = parseGraphListMembers(js, after)
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
if id.len == 0: return
- let
- headers = newHttpHeaders()
+
+ let headers = newHttpHeaders()
headers.add("Referer", """https://x.com/i/status/$1""" % id)
let
- variables = """{"rest_id":"$1"}""" % id
- params = {"variables": variables, "features": gqlFeatures}
- js = await fetch(graphTweetResult ? params, Api.tweetResult, headers)
+ url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
+ js = await fetch(url, headers)
result = parseGraphTweetResult(js)
proc getGraphTweet*(id: string; after=""): Future[Conversation] {.async.} =
if id.len == 0: return
- let
- headers = newHttpHeaders()
+
+ let headers = newHttpHeaders()
headers.add("Referer", """https://x.com/i/status/$1""" % id)
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = tweetVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures, "fieldToggles": tweetFieldToggles}
- js = await fetch(graphTweet ? params, Api.tweetDetail, headers)
+ js = await fetch(tweetDetailUrl(id, cursor), headers)
result = parseGraphConversation(js, id)
-proc getGraphFavoriters*(id: string; after=""): Future[UsersTimeline] {.async.} =
- if id.len == 0: return
- let
- cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = reactorsVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- js = await fetch(graphFavoriters ? params, Api.favoriters)
- result = parseGraphFavoritersTimeline(js, id)
-
-proc getGraphRetweeters*(id: string; after=""): Future[UsersTimeline] {.async.} =
- if id.len == 0: return
- let
- cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = reactorsVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- js = await fetch(graphRetweeters ? params, Api.retweeters)
- result = parseGraphRetweetersTimeline(js, id)
-
proc getGraphFollowing*(id: string; after=""): Future[UsersTimeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = followVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- js = await fetch(graphFollowing ? params, Api.following)
+ js = await fetch(apiReq(graphFollowing, followVars % [id, cursor]))
result = parseGraphFollowTimeline(js, id)
proc getGraphFollowers*(id: string; after=""): Future[UsersTimeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = followVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- js = await fetch(graphFollowers ? params, Api.followers)
+ js = await fetch(apiReq(graphFollowers, followVars % [id, cursor]))
result = parseGraphFollowTimeline(js, id)
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
@@ -191,8 +187,10 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
}
if after.len > 0:
variables["cursor"] = % after
- let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
- result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
+ let
+ url = apiReq(graphSearchTimeline, $variables)
+ js = await fetch(url)
+ result = parseGraphSearch[Tweets](js, after)
result.query = query
proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
@@ -211,17 +209,16 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
variables["cursor"] = % after
result.beginning = false
- let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
- result = parseGraphSearch[User](await fetch(url, Api.search), after)
+ let
+ url = apiReq(graphSearchTimeline, $variables)
+ js = await fetch(url)
+ result = parseGraphSearch[User](js, after)
result.query = query
-proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
- if name.len == 0: return
- let
- ps = genParams({"screen_name": name, "trim_user": "true"},
- count="18", ext=false)
- url = photoRail ? ps
- result = parsePhotoRail(await fetch(url, Api.photoRail))
+proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
+ if id.len == 0: return
+ let js = await fetch(mediaUrl(id, ""))
+ result = parseGraphPhotoRail(js)
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
let client = newAsyncHttpClient(maxRedirects=0)
diff --git a/src/apiutils.nim b/src/apiutils.nim
index e606ff5..535f3bd 100644
--- a/src/apiutils.nim
+++ b/src/apiutils.nim
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
import httpclient, asyncdispatch, options, strutils, uri, times, tables, math
import jsony, packedjson, zippy
-import types, tokens, consts, parserutils, http_pool
+import types, consts, parserutils, http_pool, tid
import experimental/types/common
import config
@@ -9,42 +9,34 @@ const
rlRemaining = "x-rate-limit-remaining"
rlReset = "x-rate-limit-reset"
-var pool: HttpPool
+var
+ pool: HttpPool
+ disableTid: bool
-proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
- count="20"; ext=true): seq[(string, string)] =
- result = timelineParams
- for p in pars:
- result &= p
- if ext:
- result &= ("include_ext_alt_text", "1")
- result &= ("include_ext_media_stats", "1")
- result &= ("include_ext_media_availability", "1")
- if count.len > 0:
- result &= ("count", count)
- if cursor.len > 0:
- # The raw cursor often has plus signs, which sometimes get turned into spaces,
- # so we need to turn them back into a plus
- if " " in cursor:
- result &= ("cursor", cursor.replace(" ", "+"))
- else:
- result &= ("cursor", cursor)
+proc setDisableTid*(disable: bool) =
+ disableTid = disable
+
+proc toUrl(req: ApiReq): Uri =
+ let c = req.cookie
+ parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
+
+proc rateLimitError*(): ref RateLimitError =
+ newException(RateLimitError, "rate limited")
-#proc genHeaders*(token: Token = nil): HttpHeaders =
proc genHeaders*(): HttpHeaders =
let
t = getTime()
ffVersion = floor(124 + (t.toUnix() - 1710892800) / 2419200)
result = newHttpHeaders({
"Connection": "keep-alive",
- "Authorization": auth,
+ "Authorization": bearerToken,
"Content-Type": "application/json",
"Accept-Encoding": "gzip",
"Accept-Language": "en-US,en;q=0.5",
"Accept": "*/*",
"DNT": "1",
"Host": "x.com",
- "Origin": "https://x.com/",
+ "Origin": "https://x.com",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
@@ -56,30 +48,20 @@ proc genHeaders*(): HttpHeaders =
"x-twitter-client-language": "en"
}, true)
-#template updateToken() =
-# if resp.headers.hasKey(rlRemaining):
-# let
-# remaining = parseInt(resp.headers[rlRemaining])
-# reset = parseInt(resp.headers[rlReset])
-# token.setRateLimit(api, remaining, reset)
-
template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
once:
pool = HttpPool()
- #var token = await getToken(api)
- #if token.tok.len == 0:
- # raise rateLimitError()
-
if len(cfg.cookieHeader) != 0:
additional_headers.add("Cookie", cfg.cookieHeader)
+ if not disableTid:
+ additional_headers.add("x-client-transaction-id", await genTid(url.path))
if len(cfg.xCsrfToken) != 0:
additional_headers.add("x-csrf-token", cfg.xCsrfToken)
try:
var resp: AsyncResponse
- #var headers = genHeaders(token)
var headers = genHeaders()
for key, value in additional_headers.pairs():
headers.add(key, value)
@@ -102,7 +84,6 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
- #token.setRateLimit(api, remaining, reset)
if result.len > 0:
if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -111,39 +92,27 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
if result.startsWith("{\"errors"):
let errors = result.fromJson(Errors)
if errors in {expiredToken, badToken, authorizationError}:
- echo "fetch error: ", errors
- #release(token, invalid=true)
raise rateLimitError()
elif errors in {rateLimited}:
- # rate limit hit, resets after 24 hours
- #setLimited(account, api)
raise rateLimitError()
elif result.startsWith("429 Too Many Requests"):
- #echo "[accounts] 429 error, API: ", api, ", token: ", token[]
- #account.apis[api].remaining = 0
- # rate limit hit, resets after the 15 minute window
raise rateLimitError()
fetchBody
- #release(token, used=true)
-
if resp.status == $Http400:
- let errText = "body: '" & result & "' url: " & $url
- raise newException(InternalError, errText)
+ echo "ERROR 400, ", url.path, ": ", result
+ raise newException(InternalError, $url)
except InternalError as e:
raise e
except BadClientError as e:
- #release(token, used=true)
raise e
except OSError as e:
raise e
except ProtocolError as e:
raise e
except Exception as e:
- echo "error: ", e.name, ", msg: ", e.msg, ", url: ", url
- #if "length" notin e.msg and "descriptor" notin e.msg:
- #release(token, invalid=true)
+ echo "error: ", e.name, ", msg: ", e.msg, ", url: ", url.path
raise rateLimitError()
template retry(bod) =
@@ -152,36 +121,26 @@ template retry(bod) =
except ProtocolError:
bod
-proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
+proc fetch*(req: ApiReq; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
retry:
var body: string
+ let url = req.toUrl()
fetchImpl(body, additional_headers):
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
- echo resp.status, ": ", body, " --- url: ", url
+ echo resp.status, " - non-json for: ", url, ", body: ", result
result = newJNull()
- #updateToken()
-
let error = result.getError
if error in {expiredToken, badToken}:
- echo "fetch error: ", result.getError
- #release(token, invalid=true)
+ echo "Fetch error, API: ", url.path, ", error: ", result.getError
raise rateLimitError()
-proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
+proc fetchRaw*(req: ApiReq; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
retry:
+ let url = req.toUrl()
fetchImpl(result, additional_headers):
if not (result.startsWith('{') or result.startsWith('[')):
- echo resp.status, ": ", result, " --- url: ", url
+ echo resp.status, " - non-json for: ", url, ", body: ", result
result.setLen(0)
-
- #updateToken()
-
- if result.startsWith("{\"errors"):
- let errors = result.fromJson(Errors)
- if errors in {expiredToken, badToken}:
- echo "fetch error: ", errors
- #release(token, invalid=true)
- raise rateLimitError()
diff --git a/src/auth.nim b/src/auth.nim
index de1b1d8..25648cd 100644
--- a/src/auth.nim
+++ b/src/auth.nim
@@ -126,7 +126,7 @@ proc getAccountPoolDebug*(): JsonNode =
proc rateLimitError*(): ref RateLimitError =
newException(RateLimitError, "rate limited")
-proc isLimited(account: GuestAccount; api: Api): bool =
+proc isLimited(account: GuestAccount; req: ApiReq): bool =
if account.isNil:
return true
@@ -157,9 +157,9 @@ proc release*(account: GuestAccount) =
if account.isNil: return
dec account.pending
-proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
+proc getGuestAccount*(req: ApiReq): Future[GuestAccount] {.async.} =
for i in 0 ..< accountPool.len:
- if result.isReady(api): break
+ if result.isReady(req): break
result = accountPool.sample()
if not result.isNil and result.isReady(api):
diff --git a/src/config.nim b/src/config.nim
index fe4aba5..7fdf83d 100644
--- a/src/config.nim
+++ b/src/config.nim
@@ -42,6 +42,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
enableDebug: cfg.get("Config", "enableDebug", false),
proxy: cfg.get("Config", "proxy", ""),
proxyAuth: cfg.get("Config", "proxyAuth", ""),
+ disableTid: cfg.get("Config", "disableTid", false),
cookieHeader: cfg.get("Config", "cookieHeader", ""),
xCsrfToken: cfg.get("Config", "xCsrfToken", "")
)
diff --git a/src/consts.nim b/src/consts.nim
index 7326540..e7127fd 100644
--- a/src/consts.nim
+++ b/src/consts.nim
@@ -1,54 +1,30 @@
# SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils, strutils
+import strutils
const
- auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
-
consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
+ bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
- api = parseUri("https://x.com/i/api")
- activate* = $(api / "1.1/guest/activate.json")
-
- photoRail* = api / "1.1/statuses/media_timeline.json"
-
- timelineApi = api / "2/timeline"
-
- graphql = api / "graphql"
- graphUser* = graphql / "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
- graphUserById* = graphql / "Bbaot8ySMtJD7K2t01gW7A/UserByRestId"
- graphUserTweets* = graphql / "lZRf8IC-GTuGxDwcsHW8aw/UserTweets"
- graphUserTweetsAndReplies* = graphql / "gXCeOBFsTOuimuCl1qXimg/UserTweetsAndReplies"
- graphUserMedia* = graphql / "1D04dx9H2pseMQAbMjXTvQ/UserMedia"
- graphTweet* = graphql / "6QzqakNMdh_YzBAR9SYPkQ/TweetDetail"
- graphTweetResult* = graphql / "kLXoXTloWpv9d2FSXRg-Tg/TweetResultByRestId"
- graphTweetHistory* = graphql / "WT7HhrzWulh4yudKJaR10Q/TweetEditHistory"
- graphSearchTimeline* = graphql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
- graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
- graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
- graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
- graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
- graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters"
- graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters"
- graphFollowers* = graphql / "SCu9fVIlCUm-BM8-tL5pkQ/Followers"
- graphFollowing* = graphql / "S5xUN9s2v4xk50KWGGvyvQ/Following"
- favorites* = graphql / "eSSNbhECHHWWALkkQq-YTA/Likes"
-
- timelineParams* = {
- "include_can_media_tag": "1",
- "include_cards": "1",
- "include_entities": "1",
- "include_profile_interstitial_type": "0",
- "include_quote_count": "0",
- "include_reply_count": "0",
- "include_user_entities": "0",
- "include_ext_reply_count": "0",
- "include_ext_media_color": "0",
- "cards_platform": "Web-13",
- "tweet_mode": "extended",
- "send_error_codes": "1",
- "simple_quoted_tweet": "1"
- }.toSeq
+ graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
+ graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
+ graphUserById* = "Bbaot8ySMtJD7K2t01gW7A/UserByRestId"
+ graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
+ graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
+ graphUserTweets* = "lZRf8IC-GTuGxDwcsHW8aw/UserTweets"
+ graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
+ graphUserMedia* = "1D04dx9H2pseMQAbMjXTvQ/UserMedia"
+ graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
+ graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
+ graphTweetDetail* = "6QzqakNMdh_YzBAR9SYPkQ/TweetDetail"
+ graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
+ graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
+ graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
+ graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
+ graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
+ graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
+ graphFollowers* = "Efm7xwLreAw77q2Fq7rX-Q/Followers"
+ graphFollowing* = "e0UtTAwQqgLKBllQxMgVxQ/Following"
gqlFeatures* = """{
"rweb_video_screen_enabled": false,
@@ -84,85 +60,77 @@ const
"responsive_web_grok_image_annotation_enabled": true,
"responsive_web_grok_imagine_annotation_enabled": true,
"responsive_web_grok_community_note_auto_translation_is_enabled": false,
- "responsive_web_enhance_cards_enabled": false
+ "responsive_web_enhance_cards_enabled": false,
+ "payments_enabled": false,
+ "responsive_web_twitter_article_notes_tab_enabled": false,
+ "hidden_profile_subscriptions_enabled": false,
+ "subscriptions_verification_info_verified_since_enabled": false,
+ "subscriptions_verification_info_is_identity_verified_enabled": false,
+ "highlights_tweets_tab_ui_enabled": false,
+ "subscriptions_feature_can_gift_premium": false
}""".replace(" ", "").replace("\n", "")
- userFeatures* = """{
- "hidden_profile_subscriptions_enabled": true,
- "profile_label_improvements_pcf_label_in_post_enabled": true,
- "responsive_web_profile_redirect_enabled": false,
- "rweb_tipjar_consumption_enabled": true,
- "verified_phone_label_enabled": false,
- "subscriptions_verification_info_is_identity_verified_enabled": true,
- "subscriptions_verification_info_verified_since_enabled": true,
- "highlights_tweets_tab_ui_enabled": true,
- "responsive_web_twitter_article_notes_tab_enabled": true,
- "subscriptions_feature_can_gift_premium": true,
- "creator_subscriptions_tweet_preview_api_enabled": true,
- "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
- "responsive_web_graphql_timeline_navigation_enabled": true
+ tweetVars* = """{
+ "postId": "$1",
+ $2
+ "includeHasBirdwatchNotes": false,
+ "includePromotedContent": false,
+ "withBirdwatchNotes": false,
+ "withVoice": false,
+ "withV2Timeline": true
}""".replace(" ", "").replace("\n", "")
- tweetVariables* = """{
+ tweetDetailVars* = """{
"focalTweetId": "$1",
$2
+ "referrer": "profile",
"with_rux_injections": false,
"rankingMode": "Relevance",
- "includePromotedContent": false,
+ "includePromotedContent": true,
"withCommunity": true,
- "withQuickPromoteEligibilityTweetFields": false,
+ "withQuickPromoteEligibilityTweetFields": true,
"withBirdwatchNotes": true,
"withVoice": true
}""".replace(" ", "").replace("\n", "")
- tweetFieldToggles* = """{
- "withArticleRichContentState": true,
- "withArticlePlainText": false,
- "withGrokAnalyze": false,
- "withDisallowedReplyControls": false
+ restIdVars* = """{
+ "rest_id": "$1", $2
+ "count": 20
+}"""
+
+ userMediaVars* = """{
+ "userId": "$1", $2
+ "count": 20,
+ "includePromotedContent": false,
+ "withClientEventToken": false,
+ "withBirdwatchNotes": false,
+ "withVoice": true
}""".replace(" ", "").replace("\n", "")
- userTweetsVariables* = """{
- "userId": "$1",
- $2
+ userTweetsVars* = """{
+ "userId": "$1", $2
+ "count": 20,
+ "includePromotedContent": false,
+ "withQuickPromoteEligibilityTweetFields": true,
+ "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
+ userTweetsAndRepliesVars* = """{
+ "userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withCommunity": true,
"withVoice": true
}""".replace(" ", "").replace("\n", "")
- listTweetsVariables* = """{
- "rest_id": "$1",
- $2
- "count": 20
-}""".replace(" ", "").replace("\n", "")
-
- reactorsVariables* = """{
- "tweetId": "$1",
+ followVars* = """{
+ "userId": "$1",
$2
"count": 20,
"includePromotedContent": false
}""".replace(" ", "").replace("\n", "")
- followVariables* = """{
- "userId": "$1",
- $2
- "count": 20,
- "includePromotedContent": false,
- "withGrokTranslatedBio": false
-}""".replace(" ", "").replace("\n", "")
- userMediaVariables* = """{
- "userId": "$1",
- $2
- "count": 20,
- "includePromotedContent": false,
- "withClientEventToken": false,
- "withBirdwatchNotes": true,
- "withVoice": true
-}""".replace(" ", "").replace("\n", "")
-
- tweetHistoryVariables* = """{
- "tweetId": "$1",
- "withQuickPromoteEligibilityTweetFields": false
-}""".replace(" ", "").replace("\n", "")
+ userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
+ userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
+ tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""
diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim
index 7d84833..65ebc3d 100644
--- a/src/experimental/parser/graphql.nim
+++ b/src/experimental/parser/graphql.nim
@@ -1,21 +1,53 @@
-import options
+import options, strutils
import jsony
-import user, ../types/[graphuser, graphlistmembers]
+import user, utils, ../types/[graphuser, graphlistmembers]
from ../../types import User, VerifiedType, Result, Query, QueryKind
-#proc parseGraphUser*(json: string): User =
-# if json.len == 0 or json[0] != '{':
-# return
-#
-# let raw = json.fromJson(GraphUser)
-#
-# if raw.data.userResult.result.unavailableReason.get("") == "Suspended":
-# return User(suspended: true)
-#
-# result = raw.data.userResult.result.legacy
-# result.id = raw.data.userResult.result.restId
-# if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified:
-# result.verifiedType = blue
+proc parseUserResult*(userResult: UserResult): User =
+ result = userResult.legacy
+
+ if result.verifiedType == none and userResult.isBlueVerified:
+ result.verifiedType = blue
+
+ if result.username.len == 0 and userResult.core.screenName.len > 0:
+ result.id = userResult.restId
+ result.username = userResult.core.screenName
+ result.fullname = userResult.core.name
+ result.userPic = userResult.avatar.imageUrl.replace("_normal", "")
+
+ if userResult.privacy.isSome:
+ result.protected = userResult.privacy.get.protected
+
+ if userResult.location.isSome:
+ result.location = userResult.location.get.location
+
+ if userResult.core.createdAt.len > 0:
+ result.joinDate = parseTwitterDate(userResult.core.createdAt)
+
+ if userResult.verification.isSome:
+ let v = userResult.verification.get
+ if v.verifiedType != VerifiedType.none:
+ result.verifiedType = v.verifiedType
+
+ if userResult.profileBio.isSome and result.bio.len == 0:
+ result.bio = userResult.profileBio.get.description
+
+proc parseGraphUser*(json: string): User =
+ if json.len == 0 or json[0] != '{':
+ return
+
+ let
+ raw = json.fromJson(GraphUser)
+ userResult =
+ if raw.data.userResult.isSome: raw.data.userResult.get.result
+ elif raw.data.user.isSome: raw.data.user.get.result
+ else: UserResult()
+
+ if userResult.unavailableReason.get("") == "Suspended" or
+ userResult.reason.get("") == "Suspended":
+ return User(suspended: true)
+
+ result = parseUserResult(userResult)
proc parseGraphListMembers*(json, cursor: string): Result[User] =
result = Result[User](
@@ -31,7 +63,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
of TimelineTimelineItem:
let userResult = entry.content.itemContent.userResults.result
if userResult.restId.len > 0:
- result.content.add userResult.legacy
+ result.content.add parseUserResult(userResult)
of TimelineTimelineCursor:
if entry.content.cursorType == "Bottom":
result.bottom = entry.content.value
diff --git a/src/experimental/parser/tid.nim b/src/experimental/parser/tid.nim
new file mode 100644
index 0000000..28fccea
--- /dev/null
+++ b/src/experimental/parser/tid.nim
@@ -0,0 +1,8 @@
+import jsony
+import ../types/tid
+export TidPair
+
+proc parseTidPairs*(raw: string): seq[TidPair] =
+ result = raw.fromJson(seq[TidPair])
+ if result.len == 0:
+ raise newException(ValueError, "Parsing pairs failed: " & raw)
diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim
index 08100f9..62c6612 100644
--- a/src/experimental/types/graphuser.nim
+++ b/src/experimental/types/graphuser.nim
@@ -1,15 +1,48 @@
-import options
-from ../../types import User
+import options, strutils
+from ../../types import User, VerifiedType
type
GraphUser* = object
- data*: tuple[userResult: UserData]
+ data*: tuple[userResult: Option[UserData], user: Option[UserData]]
UserData* = object
result*: UserResult
- UserResult = object
+ UserCore* = object
+ name*: string
+ screenName*: string
+ createdAt*: string
+
+ UserBio* = object
+ description*: string
+
+ UserAvatar* = object
+ imageUrl*: string
+
+ Verification* = object
+ verifiedType*: VerifiedType
+
+ Location* = object
+ location*: string
+
+ Privacy* = object
+ protected*: bool
+
+ UserResult* = object
legacy*: User
restId*: string
isBlueVerified*: bool
+ core*: UserCore
+ avatar*: UserAvatar
unavailableReason*: Option[string]
+ reason*: Option[string]
+ privacy*: Option[Privacy]
+ profileBio*: Option[UserBio]
+ verification*: Option[Verification]
+ location*: Option[Location]
+
+proc enumHook*(s: string; v: var VerifiedType) =
+ v = try:
+ parseEnum[VerifiedType](s)
+ except:
+ VerifiedType.none
diff --git a/src/experimental/types/tid.nim b/src/experimental/types/tid.nim
new file mode 100644
index 0000000..ad036d9
--- /dev/null
+++ b/src/experimental/types/tid.nim
@@ -0,0 +1,4 @@
+type
+ TidPair* = object
+ animationKey*: string
+ verification*: string
diff --git a/src/nitter.nim b/src/nitter.nim
index a813490..0db3980 100644
--- a/src/nitter.nim
+++ b/src/nitter.nim
@@ -6,7 +6,7 @@ from htmlgen import a
import jester
-import types, config, prefs, formatters, redis_cache, http_pool
+import types, config, prefs, formatters, redis_cache, http_pool, apiutils
import views/[general, about]
import routes/[
preferences, timeline, status, media, search, list, rss, #debug,
@@ -34,6 +34,7 @@ setHmacKey(cfg.hmacKey)
setProxyEncoding(cfg.base64Media)
setMaxHttpConns(cfg.httpMaxConns)
setHttpProxy(cfg.proxy, cfg.proxyAuth)
+setDisableTid(cfg.disableTid)
initAboutPage(cfg.staticDir)
waitFor initRedisPool(cfg)
@@ -104,7 +105,6 @@ routes:
extend preferences, ""
extend resolver, ""
extend embed, ""
- #extend debug, ""
extend activityspoof, ""
extend api, ""
extend unsupported, ""
diff --git a/src/parser.nim b/src/parser.nim
index f10fc7e..44e0a37 100644
--- a/src/parser.nim
+++ b/src/parser.nim
@@ -45,6 +45,7 @@ proc parseGraphUser*(js: JsonNode): User =
result.fullname = user{"core", "name"}.getStr
result.joinDate = parseTwitterDate(user{"core", "created_at"}.getStr)
result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
+ result.protected = user{"privacy", "protected"}.getBool
let label = user{"affiliates_highlighted_label", "label"}
if not label.isNull:
@@ -449,23 +450,6 @@ proc parseTimeline*(js: JsonNode; after=""): Profile =
else:
result.tweets.top = cursor{"value"}.getStr
-proc parsePhotoRail*(js: JsonNode): PhotoRail =
- with error, js{"error"}:
- if error.getStr == "Not authorized.":
- return
-
- for tweet in js:
- let
- t = parseTweet(tweet, js{"tweet_card"})
- url = if t.photos.len > 0: t.photos[0].url
- elif t.video.isSome: get(t.video).thumb
- elif t.gif.isSome: get(t.gif).thumb
- elif t.card.isSome: get(t.card).image
- else: ""
-
- if url.len == 0: continue
- result.add GalleryPhoto(url: url, tweetId: $t.id)
-
proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
if js.kind == JNull:
return Tweet()
@@ -531,6 +515,34 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
else:
result.stats.views = -1
+proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
+ result = @[]
+
+ let instructions =
+ ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
+
+ for i in instructions:
+ if i{"__typename"}.getStr == "TimelineAddEntries":
+ for e in i{"entries"}:
+ let entryId = e{"entryId"}.getStr
+ if entryId.startsWith("tweet"):
+ with tweetResult, e{"content", "content", "tweetResult", "result"}:
+ let t = parseGraphTweet(tweetResult, false)
+ if not t.available:
+ t.id = $parseBiggestInt(entryId.getId())
+
+ let url =
+ if t.photos.len > 0: t.photos[0].url
+ elif t.video.isSome: get(t.video).thumb
+ elif t.gif.isSome: get(t.gif).thumb
+ elif t.card.isSome: get(t.card).image
+ else: ""
+
+ result.add GalleryPhoto(url: url, tweetId: t.id)
+
+ if result.len == 16:
+ break
+
proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
for t in js{"content", "items"}:
let entryId = t{"entryId"}.getStr
diff --git a/src/redis_cache.nim b/src/redis_cache.nim
index 16f3c25..13b99e3 100644
--- a/src/redis_cache.nim
+++ b/src/redis_cache.nim
@@ -87,7 +87,7 @@ proc cache*(data: List) {.async.} =
await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
proc cache*(data: PhotoRail; name: string) {.async.} =
- await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
+ await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
proc cache*(data: User) {.async.} =
if data.username.len == 0: return
@@ -145,6 +145,9 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
elif fetch:
result = await getGraphUser(username)
await cache(result)
+ if result.id.len > 0:
+ await setEx("i:" & result.id, baseCacheTime, result.username)
+ await cacheUserId(result.username, result.id)
proc getCachedUsername*(userId: string): Future[string] {.async.} =
let
@@ -174,14 +177,14 @@ proc getCachedTweet*(id: string; after=""): Future[Conversation] {.async.} =
if not result.isNil and after.len > 0:
result.replies = await getReplies(id, after)
-proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
- if name.len == 0: return
- let rail = await get("pr:" & toLower(name))
+proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
+ if id.len == 0: return
+ let rail = await get("pr2:" & toLower(id))
if rail != redisNil:
rail.deserialize(PhotoRail)
else:
- result = await getPhotoRail(name)
- await cache(result, name)
+ result = await getPhotoRail(id)
+ await cache(result, id)
proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
let list = if id.len == 0: redisNil
diff --git a/src/routes/activityspoof.nim b/src/routes/activityspoof.nim
index c5ca98e..08f7e04 100644
--- a/src/routes/activityspoof.nim
+++ b/src/routes/activityspoof.nim
@@ -19,14 +19,15 @@ proc createActivityPubRouter*(cfg: Config) =
get "/api/v1/accounts/@id":
let id = @"id"
- if id.len > 19 or id.any(c => not c.isDigit):
+ #if id.len > 19 or id.any(c => not c.isDigit):
+ if not id.allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}):
resp Http404, {"Content-Type": "application/json"}, """{"error":"Invalid account ID"}"""
- var username = await getCachedUsername(id)
- if username.len == 0:
- resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
+ #var username = await getCachedUsername(id)
+ #if username.len == 0:
+ #resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
- let user = await getCachedUser(username)
+ let user = await getCachedUser(id)
if user.suspended or user.id.len == 0:
resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
@@ -189,9 +190,8 @@ proc createActivityPubRouter*(cfg: Config) =
postJson["edited_at"] = newJNull()
postJson["reblog"] = newJNull()
if tweet.replyId.len != 0:
- let replyUser = await getCachedUser(tweet.replyHandle)
postJson["in_reply_to_id"] = %(&"{tweet.replyId}")
- postJson["in_reply_to_account_id"] = %replyUser.id
+ postJson["in_reply_to_account_id"] = %tweet.replyHandle
else:
postJson["in_reply_to_id"] = newJNull()
postJson["in_reply_to_account_id"] = newJNull()
diff --git a/src/routes/debug.nim b/src/routes/debug.nim
deleted file mode 100644
index 47a8b22..0000000
--- a/src/routes/debug.nim
+++ /dev/null
@@ -1,17 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import jester
-import router_utils
-import ".."/[tokens, types]
-
-proc createDebugRouter*(cfg: Config) =
- router debug:
- get "/.tokens":
- cond cfg.enableDebug
- respJson getPoolJson()
-
- #get "/.health":
- #respJson getAccountPoolHealth()
-
- #get "/.accounts":
- #cond cfg.enableDebug
- #respJson getAccountPoolDebug()
diff --git a/src/routes/status.nim b/src/routes/status.nim
index f7c69bf..3516a19 100644
--- a/src/routes/status.nim
+++ b/src/routes/status.nim
@@ -5,7 +5,7 @@ import jester, karax/vdom
import router_utils
import ".."/[types, formatters, api, redis_cache]
-import ../views/[general, status, search, mastoapi]
+import ../views/[general, status, mastoapi]
export json, uri, sequtils, options, sugar, times
export router_utils
diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim
index a8967d9..d589687 100644
--- a/src/routes/timeline.nim
+++ b/src/routes/timeline.nim
@@ -48,7 +48,7 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
let
rail =
skipIf(skipRail or query.kind == media, @[]):
- getCachedPhotoRail(name)
+ getCachedPhotoRail(userId)
user = getCachedUser(name)
@@ -111,6 +111,7 @@ proc createTimelineRouter*(cfg: Config) =
get "/@name/?@tab?/?":
cond '.' notin @"name"
cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
+ cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_', ','})
cond @"tab" in ["with_replies", "media", "search", "following", "followers", ""]
let
prefs = cookiePrefs()
diff --git a/src/routes/twitter_api.nim b/src/routes/twitter_api.nim
index b58924b..5a04022 100644
--- a/src/routes/twitter_api.nim
+++ b/src/routes/twitter_api.nim
@@ -4,7 +4,7 @@ import json, asyncdispatch, options, uri
import times
import jester
import router_utils
-import ".."/[types, api, apiutils, query, consts, redis_cache]
+import ".."/[types, api, apiutils, redis_cache]
import httpclient, strutils
import sequtils
@@ -81,60 +81,49 @@ proc getUserTweetsJson*(id: string): Future[JsonNode] {.async.} =
result = response
-proc searchTimeline*(query: Query; after=""): Future[string] {.async.} =
- let q = genQueryParam(query)
- var
- variables = %*{
- "rawQuery": q,
- "count": 20,
- "product": "Latest",
- "withDownvotePerspective": false,
- "withReactionsMetadata": false,
- "withReactionsPerspective": false
- }
- if after.len > 0:
- variables["cursor"] = % after
- let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
- result = await fetchRaw(url, Api.search)
-
proc getUserTweets*(id: string; after=""): Future[string] {.async.} =
if id.len == 0: return
- let
- cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = userTweetsVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- result = await fetchRaw(graphUserTweets ? params, Api.userTweets)
+
+ let headers = newHttpHeaders()
+ headers.add("Referer", """https://x.com/$1""" % id)
+
+ let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+ result = await fetchRaw(userTweetsUrl(id, cursor), headers)
proc getUserReplies*(id: string; after=""): Future[string] {.async.} =
if id.len == 0: return
- let
- cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = userTweetsVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- result = await fetchRaw(graphUserTweets ? params, Api.userTweetsAndReplies)
+
+ let headers = newHttpHeaders()
+ headers.add("Referer", """https://x.com/$1/with_replies""" % id)
+
+ let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+ result = await fetchRaw(userTweetsAndRepliesUrl(id, cursor), headers)
proc getUserMedia*(id: string; after=""): Future[string] {.async.} =
if id.len == 0: return
- let
- cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = userMediaVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- result = await fetchRaw(graphUserMedia ? params, Api.userMedia)
+
+ let headers = newHttpHeaders()
+ headers.add("Referer", """https://x.com/$1/media""" % id)
+
+ let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+ result = await fetchRaw(mediaUrl(id, cursor), headers)
proc getTweetById*(id: string; after=""): Future[string] {.async.} =
- let
- cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = tweetVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- result = await fetchRaw(graphTweet ? params, Api.tweetDetail)
+ if id.len == 0: return
+
+ let headers = newHttpHeaders()
+ headers.add("Referer", """https://x.com/i/status/$1""" % id)
+
+ let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+ result = await fetchRaw(tweetDetailUrl(id, cursor), headers)
proc getUser*(username: string): Future[string] {.async.} =
if username.len == 0: return
- let
- variables = """{"screen_name":"$1"}""" % username
- fieldToggles = """{"withAuxiliaryUserLabels":true}"""
- params = {"variables": variables, "features": gqlFeatures, "fieldToggles": fieldToggles}
- result = await fetchRaw(graphUser ? params, Api.userScreenName)
+
+ let headers = newHttpHeaders()
+ headers.add("Referer", """https://x.com/$1""" % username)
+
+ result = await fetchRaw(userUrl(username), headers)
proc createTwitterApiRouter*(cfg: Config) =
router api:
@@ -146,16 +135,11 @@ proc createTwitterApiRouter*(cfg: Config) =
let response = await getUser(username)
resp Http200, { "Content-Type": "application/json" }, response
- #get "/api/user/@id/tweets":
- # let id = @"id"
- # let response = await getUserTweetsJson(id)
- # respJson response
-
- get "/api/user/@username/timeline":
- let username = @"username"
- let query = Query(fromUser: @[username])
- let response = await searchTimeline(query)
- resp Http200, { "Content-Type": "application/json" }, response
+ #get "/api/user/@username/timeline":
+ # let username = @"username"
+ # let query = Query(fromUser: @[username])
+ # let response = await searchTimeline(query)
+ # resp Http200, { "Content-Type": "application/json" }, response
get "/api/user/@id/tweets":
let id = @"id"
diff --git a/src/sass/general.scss b/src/sass/general.scss
index debf89f..2064213 100644
--- a/src/sass/general.scss
+++ b/src/sass/general.scss
@@ -23,6 +23,7 @@
font-weight: bold;
width: 30px;
height: 30px;
+ padding: 0px 5px 1px 8px;
}
input {
diff --git a/src/sass/index.scss b/src/sass/index.scss
index 6cab48e..94582ce 100644
--- a/src/sass/index.scss
+++ b/src/sass/index.scss
@@ -1,180 +1,202 @@
-@import '_variables';
+@import "_variables";
-@import 'tweet/_base';
-@import 'profile/_base';
-@import 'general';
-@import 'navbar';
-@import 'inputs';
-@import 'timeline';
-@import 'search';
+@import "tweet/_base";
+@import "profile/_base";
+@import "general";
+@import "navbar";
+@import "inputs";
+@import "timeline";
+@import "search";
body {
- // colors
- --bg_color: #{$bg_color};
- --fg_color: #{$fg_color};
- --fg_faded: #{$fg_faded};
- --fg_dark: #{$fg_dark};
- --fg_nav: #{$fg_nav};
+ // colors
+ --bg_color: #{$bg_color};
+ --fg_color: #{$fg_color};
+ --fg_faded: #{$fg_faded};
+ --fg_dark: #{$fg_dark};
+ --fg_nav: #{$fg_nav};
- --bg_panel: #{$bg_panel};
- --bg_elements: #{$bg_elements};
- --bg_overlays: #{$bg_overlays};
- --bg_hover: #{$bg_hover};
+ --bg_panel: #{$bg_panel};
+ --bg_elements: #{$bg_elements};
+ --bg_overlays: #{$bg_overlays};
+ --bg_hover: #{$bg_hover};
- --grey: #{$grey};
- --dark_grey: #{$dark_grey};
- --darker_grey: #{$darker_grey};
- --darkest_grey: #{$darkest_grey};
- --border_grey: #{$border_grey};
+ --grey: #{$grey};
+ --dark_grey: #{$dark_grey};
+ --darker_grey: #{$darker_grey};
+ --darkest_grey: #{$darkest_grey};
+ --border_grey: #{$border_grey};
- --accent: #{$accent};
- --accent_light: #{$accent_light};
- --accent_dark: #{$accent_dark};
- --accent_border: #{$accent_border};
+ --accent: #{$accent};
+ --accent_light: #{$accent_light};
+ --accent_dark: #{$accent_dark};
+ --accent_border: #{$accent_border};
- --play_button: #{$play_button};
- --play_button_hover: #{$play_button_hover};
+ --play_button: #{$play_button};
+ --play_button_hover: #{$play_button_hover};
- --more_replies_dots: #{$more_replies_dots};
- --error_red: #{$error_red};
+ --more_replies_dots: #{$more_replies_dots};
+ --error_red: #{$error_red};
- --verified_blue: #{$verified_blue};
- --verified_business: #{$verified_business};
- --verified_government: #{$verified_government};
- --icon_text: #{$icon_text};
+ --verified_blue: #{$verified_blue};
+ --verified_business: #{$verified_business};
+ --verified_government: #{$verified_government};
+ --icon_text: #{$icon_text};
- --tab: #{$fg_color};
- --tab_selected: #{$accent};
+ --tab: #{$fg_color};
+ --tab_selected: #{$accent};
- --profile_stat: #{$fg_color};
+ --profile_stat: #{$fg_color};
- background-color: var(--bg_color);
- color: var(--fg_color);
- font-family: $font_0, $font_1, $font_2, $font_3;
- font-size: 14px;
- line-height: 1.3;
- margin: 0;
+ background-color: var(--bg_color);
+ color: var(--fg_color);
+ font-family: $font_0, $font_1;
+ font-size: 14px;
+ line-height: 1.3;
+ margin: 0;
}
* {
- outline: unset;
- margin: 0;
- text-decoration: none;
+ outline: unset;
+ margin: 0;
+ text-decoration: none;
}
h1 {
- display: inline;
+ display: inline;
}
-h2, h3 {
- font-weight: normal;
+h2,
+h3 {
+ font-weight: normal;
}
p {
- margin: 14px 0;
+ margin: 14px 0;
}
a {
- color: var(--accent);
+ color: var(--accent);
- &:hover {
- text-decoration: underline;
- }
+ &:hover {
+ text-decoration: underline;
+ }
}
fieldset {
- border: 0;
- padding: 0;
- margin-top: -0.6em;
+ border: 0;
+ padding: 0;
+ margin-top: -0.6em;
}
legend {
- width: 100%;
- padding: .6em 0 .3em 0;
- border: 0;
- font-size: 16px;
- font-weight: 600;
- border-bottom: 1px solid var(--border_grey);
- margin-bottom: 8px;
+ width: 100%;
+ padding: 0.6em 0 0.3em 0;
+ border: 0;
+ font-size: 16px;
+ font-weight: 600;
+ border-bottom: 1px solid var(--border_grey);
+ margin-bottom: 8px;
}
.preferences .note {
- border-top: 1px solid var(--border_grey);
- border-bottom: 1px solid var(--border_grey);
- padding: 6px 0 8px 0;
- margin-bottom: 8px;
- margin-top: 16px;
+ border-top: 1px solid var(--border_grey);
+ border-bottom: 1px solid var(--border_grey);
+ padding: 6px 0 8px 0;
+ margin-bottom: 8px;
+ margin-top: 16px;
}
ul {
- padding-left: 1.3em;
+ padding-left: 1.3em;
}
.container {
- display: flex;
- flex-wrap: wrap;
- box-sizing: border-box;
- padding-top: 50px;
- margin: auto;
- min-height: 100vh;
+ display: flex;
+ flex-wrap: wrap;
+ box-sizing: border-box;
+ padding-top: 50px;
+ margin: auto;
+ min-height: 100vh;
}
.icon-container {
- display: inline;
+ display: inline;
}
.overlay-panel {
- max-width: 600px;
- width: 100%;
- margin: 0 auto;
- margin-top: 10px;
- background-color: var(--bg_overlays);
- padding: 10px 15px;
- align-self: start;
+ max-width: 600px;
+ width: 100%;
+ margin: 0 auto;
+ margin-top: 10px;
+ background-color: var(--bg_overlays);
+ padding: 10px 15px;
+ align-self: start;
- ul {
- margin-bottom: 14px;
- }
+ ul {
+ margin-bottom: 14px;
+ }
- p {
- word-break: break-word;
- }
+ p {
+ word-break: break-word;
+ }
}
.verified-icon {
- color: var(--icon_text);
- border-radius: 50%;
- flex-shrink: 0;
- margin: 2px 0 3px 3px;
- padding-top: 3px;
- height: 11px;
- width: 14px;
- font-size: 8px;
- display: inline-block;
- text-align: center;
- vertical-align: middle;
+ display: inline-block;
+ width: 14px;
+ height: 14px;
+ margin-left: 2px;
- &.blue {
- background-color: var(--verified_blue);
+ .verified-icon-circle {
+ position: absolute;
+ font-size: 15px;
+ }
+
+ .verified-icon-check {
+ position: absolute;
+ font-size: 9px;
+ margin: 5px 3px;
+ }
+
+ &.blue {
+ .verified-icon-circle {
+ color: var(--verified_blue);
}
- &.business {
- color: var(--bg_panel);
- background-color: var(--verified_business);
+ .verified-icon-check {
+ color: var(--icon_text);
+ }
+ }
+
+ &.business {
+ .verified-icon-circle {
+ color: var(--verified_business);
}
- &.government {
- color: var(--bg_panel);
- background-color: var(--verified_government);
+ .verified-icon-check {
+ color: var(--bg_panel);
}
+ }
+
+ &.government {
+ .verified-icon-circle {
+ color: var(--verified_government);
+ }
+
+ .verified-icon-check {
+ color: var(--bg_panel);
+ }
+ }
}
-@media(max-width: 600px) {
- .preferences-container {
- max-width: 95vw;
- }
+@media (max-width: 600px) {
+ .preferences-container {
+ max-width: 95vw;
+ }
- .nav-item, .nav-item .icon-container {
- font-size: 16px;
- }
+ .nav-item,
+ .nav-item .icon-container {
+ font-size: 16px;
+ }
}
diff --git a/src/sass/navbar.scss b/src/sass/navbar.scss
index 47a8765..86bfbe7 100644
--- a/src/sass/navbar.scss
+++ b/src/sass/navbar.scss
@@ -1,89 +1,87 @@
-@import '_variables';
+@import "_variables";
nav {
- display: flex;
- align-items: center;
- position: fixed;
- background-color: var(--bg_overlays);
- box-shadow: 0 0 4px $shadow;
- padding: 0;
- width: 100%;
- height: 50px;
- z-index: 1000;
- font-size: 16px;
+ display: flex;
+ align-items: center;
+ position: fixed;
+ background-color: var(--bg_overlays);
+ box-shadow: 0 0 4px $shadow;
+ padding: 0;
+ width: 100%;
+ height: 50px;
+ z-index: 1000;
+ font-size: 16px;
- a, .icon-button button {
- color: var(--fg_nav);
- }
+ a,
+ .icon-button button {
+ color: var(--fg_nav);
+ }
}
.inner-nav {
- margin: auto;
- box-sizing: border-box;
- padding: 0 10px;
- display: flex;
- align-items: center;
- flex-basis: 920px;
- height: 50px;
+ margin: auto;
+ box-sizing: border-box;
+ padding: 0 10px;
+ display: flex;
+ align-items: center;
+ flex-basis: 920px;
+ height: 50px;
}
.site-name {
- font-size: 15px;
- font-weight: 600;
- line-height: 1;
+ font-size: 15px;
+ font-weight: 600;
+ line-height: 1;
- &:hover {
- color: var(--accent_light);
- text-decoration: unset;
- }
+ &:hover {
+ color: var(--accent_light);
+ text-decoration: unset;
+ }
}
.site-logo {
- display: block;
- width: 35px;
- height: 35px;
+ display: block;
+ width: 35px;
+ height: 35px;
}
.nav-item {
- display: flex;
- flex: 1;
- line-height: 50px;
- height: 50px;
- overflow: hidden;
- flex-wrap: wrap;
- align-items: center;
+ display: flex;
+ flex: 1;
+ line-height: 50px;
+ height: 50px;
+ overflow: hidden;
+ flex-wrap: wrap;
+ align-items: center;
- &.right {
- text-align: right;
- justify-content: flex-end;
- }
+ &.right {
+ text-align: right;
+ justify-content: flex-end;
+ }
- &.right a {
- padding-left: 4px;
-
- &:hover {
- color: var(--accent_light);
- text-decoration: unset;
- }
- }
+ &.right a:hover {
+ color: var(--accent_light);
+ text-decoration: unset;
+ }
}
.lp {
- height: 14px;
- display: inline-block;
- position: relative;
- top: 2px;
- fill: var(--fg_nav);
+ height: 14px;
+ display: inline-block;
+ position: relative;
+ top: 2px;
+ fill: var(--fg_nav);
- &:hover {
- fill: var(--accent_light);
- }
+ &:hover {
+ fill: var(--accent_light);
+ }
}
-.icon-info:before {
- margin: 0 -3px;
+.icon-info {
+ margin: 0 -3px;
}
.icon-cog {
- font-size: 15px;
+ font-size: 15px;
+ padding-left: 0 !important;
}
diff --git a/src/sass/search.scss b/src/sass/search.scss
index f70f7ea..458290f 100644
--- a/src/sass/search.scss
+++ b/src/sass/search.scss
@@ -1,120 +1,121 @@
-@import '_variables';
-@import '_mixins';
+@import "_variables";
+@import "_mixins";
.search-title {
- font-weight: bold;
- display: inline-block;
- margin-top: 4px;
+ font-weight: bold;
+ display: inline-block;
+ margin-top: 4px;
}
.search-field {
+ display: flex;
+ flex-wrap: wrap;
+
+ button {
+ margin: 0 2px 0 0;
+ padding: 0px 1px 1px 4px;
+ height: 23px;
display: flex;
- flex-wrap: wrap;
+ align-items: center;
+ }
- button {
- margin: 0 2px 0 0;
- height: 23px;
- display: flex;
- align-items: center;
- }
+ .pref-input {
+ margin: 0 4px 0 0;
+ flex-grow: 1;
+ height: 23px;
+ }
- .pref-input {
- margin: 0 4px 0 0;
- flex-grow: 1;
- height: 23px;
- }
+ input[type="text"] {
+ height: calc(100% - 4px);
+ width: calc(100% - 8px);
+ }
- input[type="text"] {
- height: calc(100% - 4px);
- width: calc(100% - 8px);
- }
+ > label {
+ display: inline;
+ background-color: var(--bg_elements);
+ color: var(--fg_color);
+ border: 1px solid var(--accent_border);
+ padding: 1px 1px 2px 4px;
+ font-size: 14px;
+ cursor: pointer;
+ margin-bottom: 2px;
- > label {
- display: inline;
- background-color: var(--bg_elements);
- color: var(--fg_color);
- border: 1px solid var(--accent_border);
- padding: 1px 6px 2px 6px;
- font-size: 14px;
- cursor: pointer;
- margin-bottom: 2px;
+ @include input-colors;
+ }
- @include input-colors;
- }
-
- @include create-toggle(search-panel, 200px);
+ @include create-toggle(search-panel, 200px);
}
.search-panel {
- width: 100%;
- max-height: 0;
- overflow: hidden;
- transition: max-height 0.4s;
+ width: 100%;
+ max-height: 0;
+ overflow: hidden;
+ transition: max-height 0.4s;
- flex-grow: 1;
- font-weight: initial;
- text-align: left;
+ flex-grow: 1;
+ font-weight: initial;
+ text-align: left;
- > div {
- line-height: 1.7em;
- }
+ > div {
+ line-height: 1.7em;
+ }
- .checkbox-container {
- display: inline;
- padding-right: unset;
- margin-bottom: unset;
- margin-left: 23px;
- }
+ .checkbox-container {
+ display: inline;
+ padding-right: unset;
+ margin-bottom: 5px;
+ margin-left: 23px;
+ }
- .checkbox {
- right: unset;
- left: -22px;
- }
+ .checkbox {
+ right: unset;
+ left: -22px;
+ }
- .checkbox-container .checkbox:after {
- top: -4px;
- }
+ .checkbox-container .checkbox:after {
+ top: -4px;
+ }
}
.search-row {
- display: flex;
- flex-wrap: wrap;
- line-height: unset;
+ display: flex;
+ flex-wrap: wrap;
+ line-height: unset;
- > div {
- flex-grow: 1;
- flex-shrink: 1;
- }
+ > div {
+ flex-grow: 1;
+ flex-shrink: 1;
+ }
+
+ input {
+ height: 21px;
+ }
+
+ .pref-input {
+ display: block;
+ padding-bottom: 5px;
input {
- height: 21px;
- }
-
- .pref-input {
- display: block;
- padding-bottom: 5px;
-
- input {
- height: 21px;
- margin-top: 1px;
- }
+ height: 21px;
+ margin-top: 1px;
}
+ }
}
.search-toggles {
- flex-grow: 1;
- display: grid;
- grid-template-columns: repeat(6, auto);
- grid-column-gap: 10px;
+ flex-grow: 1;
+ display: grid;
+ grid-template-columns: repeat(6, auto);
+ grid-column-gap: 10px;
}
.profile-tabs {
- @include search-resize(820px, 5);
- @include search-resize(725px, 4);
- @include search-resize(600px, 6);
- @include search-resize(560px, 5);
- @include search-resize(480px, 4);
- @include search-resize(410px, 3);
+ @include search-resize(820px, 5);
+ @include search-resize(725px, 4);
+ @include search-resize(600px, 6);
+ @include search-resize(560px, 5);
+ @include search-resize(480px, 4);
+ @include search-resize(410px, 3);
}
@include search-resize(560px, 5);
diff --git a/src/sass/tweet/video.scss b/src/sass/tweet/video.scss
index 1e9096e..37bab98 100644
--- a/src/sass/tweet/video.scss
+++ b/src/sass/tweet/video.scss
@@ -1,66 +1,64 @@
-@import '_variables';
-@import '_mixins';
+@import "_variables";
+@import "_mixins";
video {
- max-height: 100%;
- width: 100%;
+ height: 100%;
+ width: 100%;
}
.gallery-video {
- display: flex;
- overflow: hidden;
+ display: flex;
+ overflow: hidden;
}
.gallery-video.card-container {
- flex-direction: column;
+ flex-direction: column;
+ width: 100%;
}
.video-container {
- max-height: 530px;
- margin: 0;
- display: flex;
- align-items: center;
- justify-content: center;
+ max-height: 530px;
+ margin: 0;
- img {
- max-height: 100%;
- max-width: 100%;
- }
+ img {
+ max-height: 100%;
+ max-width: 100%;
+ }
}
.video-overlay {
- @include play-button;
- background-color: $shadow;
+ @include play-button;
+ background-color: $shadow;
- p {
- position: relative;
- z-index: 0;
- text-align: center;
- top: calc(50% - 20px);
- font-size: 20px;
- line-height: 1.3;
- margin: 0 20px;
- }
+ p {
+ position: relative;
+ z-index: 0;
+ text-align: center;
+ top: calc(50% - 20px);
+ font-size: 20px;
+ line-height: 1.3;
+ margin: 0 20px;
+ }
- div {
- position: relative;
- z-index: 0;
- top: calc(50% - 20px);
- margin: 0 auto;
- width: 40px;
- height: 40px;
- }
+ div {
+ position: relative;
+ z-index: 0;
+ top: calc(50% - 20px);
+ margin: 0 auto;
+ width: 40px;
+ height: 40px;
+ }
- form {
- width: 100%;
- height: 100%;
- align-items: center;
- justify-content: center;
- display: flex;
- }
+ form {
+ width: 100%;
+ height: 100%;
+ align-items: center;
+ justify-content: center;
+ display: flex;
+ }
- button {
- padding: 5px 8px;
- font-size: 16px;
- }
+ button {
+ padding: 5px 8px;
+ font-size: 16px;
+ }
}
diff --git a/src/tid.nim b/src/tid.nim
new file mode 100644
index 0000000..7b453fb
--- /dev/null
+++ b/src/tid.nim
@@ -0,0 +1,62 @@
+import std/[asyncdispatch, base64, httpclient, random, strutils, sequtils, times]
+import nimcrypto
+import experimental/parser/tid
+
+randomize()
+
+const defaultKeyword = "obfiowerehiring";
+const pairsUrl =
+ "https://raw.githubusercontent.com/fa0311/x-client-transaction-id-pair-dict/refs/heads/main/pair.json";
+
+var
+ cachedPairs: seq[TidPair] = @[]
+ lastCached = 0
+ # refresh every hour
+ ttlSec = 60 * 60
+
+proc getPair(): Future[TidPair] {.async.} =
+ if cachedPairs.len == 0 or int(epochTime()) - lastCached > ttlSec:
+ lastCached = int(epochTime())
+
+ let client = newAsyncHttpClient()
+ defer: client.close()
+
+ let resp = await client.get(pairsUrl)
+ if resp.status == $Http200:
+ cachedPairs = parseTidPairs(await resp.body)
+
+ return sample(cachedPairs)
+
+proc encodeSha256(text: string): array[32, byte] =
+ let
+ data = cast[ptr byte](addr text[0])
+ dataLen = uint(len(text))
+ digest = sha256.digest(data, dataLen)
+ return digest.data
+
+proc encodeBase64[T](data: T): string =
+ return encode(data).replace("=", "")
+
+proc decodeBase64(data: string): seq[byte] =
+ return cast[seq[byte]](decode(data))
+
+proc genTid*(path: string): Future[string] {.async.} =
+ let
+ pair = await getPair()
+
+ timeNow = int(epochTime() - 1682924400)
+ timeNowBytes = @[
+ byte(timeNow and 0xff),
+ byte((timeNow shr 8) and 0xff),
+ byte((timeNow shr 16) and 0xff),
+ byte((timeNow shr 24) and 0xff)
+ ]
+
+ data = "GET!" & path & "!" & $timeNow & defaultKeyword & pair.animationKey
+ hashBytes = encodeSha256(data)
+ keyBytes = decodeBase64(pair.verification)
+ bytesArr = keyBytes & timeNowBytes & hashBytes[0 ..< 16] & @[3'u8]
+ randomNum = byte(rand(256))
+ tid = @[randomNum] & bytesArr.mapIt(it xor randomNum)
+
+ return encodeBase64(tid)
diff --git a/src/tokens.nim b/src/tokens.nim
deleted file mode 100644
index f622e4d..0000000
--- a/src/tokens.nim
+++ /dev/null
@@ -1,168 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, times, sequtils, json, random
-import strutils, tables
-import types, consts
-
-const
- maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
- maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
- maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
- failDelay = initDuration(minutes=30)
-
-var
- tokenPool: seq[Token]
- lastFailed: Time
- enableLogging = false
-
-let headers = newHttpHeaders({"authorization": auth})
-
-template log(str) =
- if enableLogging: echo "[tokens] ", str
-
-proc getPoolJson*(): JsonNode =
- var
- list = newJObject()
- totalReqs = 0
- totalPending = 0
- reqsPerApi: Table[string, int]
-
- for token in tokenPool:
- totalPending.inc(token.pending)
- list[token.tok] = %*{
- "apis": newJObject(),
- "pending": token.pending,
- "init": $token.init,
- "lastUse": $token.lastUse
- }
-
- for api in token.apis.keys:
- list[token.tok]["apis"][$api] = %token.apis[api]
-
- let
- maxReqs =
- case api
- of Api.photoRail: 180
- #of Api.timeline: 187
- #of Api.userTweets, Api.userTimeline: 300
- of Api.userTweets: 300
- of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
- Api.userTweetsAndReplies, Api.userMedia,
- Api.userRestId, Api.userScreenName, Api.tweetDetail,
- Api.tweetResult, Api.search, Api.favorites,
- Api.retweeters, Api.favoriters, Api.following, Api.followers: 500
- #of Api.userSearch: 900
- reqs = maxReqs - token.apis[api].remaining
-
- reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
- totalReqs.inc(reqs)
-
- return %*{
- "amount": tokenPool.len,
- "requests": totalReqs,
- "pending": totalPending,
- "apis": reqsPerApi,
- "tokens": list
- }
-
-proc rateLimitError*(): ref RateLimitError =
- newException(RateLimitError, "rate limited")
-
-proc fetchToken(): Future[Token] {.async.} =
- if getTime() - lastFailed < failDelay:
- raise rateLimitError()
-
- let client = newAsyncHttpClient(headers=headers)
-
- try:
- let
- resp = await client.postContent(activate)
- tokNode = parseJson(resp)["guest_token"]
- tok = tokNode.getStr($(tokNode.getInt))
- time = getTime()
-
- return Token(tok: tok, init: time, lastUse: time)
- except Exception as e:
- echo "[tokens] fetching token failed: ", e.msg
- if "Try again" notin e.msg:
- echo "[tokens] fetching tokens paused, resuming in 30 minutes"
- lastFailed = getTime()
- finally:
- client.close()
-
-proc expired(token: Token): bool =
- let time = getTime()
- token.init < time - maxAge or token.lastUse < time - maxLastUse
-
-proc isLimited(token: Token; api: Api): bool =
- if token.isNil or token.expired:
- return true
-
- if api in token.apis:
- let limit = token.apis[api]
- return (limit.remaining <= 10 and limit.reset > epochTime().int)
- else:
- return false
-
-proc isReady(token: Token; api: Api): bool =
- not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
-
-proc release*(token: Token; used=false; invalid=false) =
- if token.isNil: return
- if invalid or token.expired:
- if invalid: log "discarding invalid token"
- elif token.expired: log "discarding expired token"
-
- let idx = tokenPool.find(token)
- if idx > -1: tokenPool.delete(idx)
- elif used:
- dec token.pending
- token.lastUse = getTime()
-
-proc getToken*(api: Api): Future[Token] {.async.} =
- for i in 0 ..< tokenPool.len:
- if result.isReady(api): break
- release(result)
- result = tokenPool.sample()
-
- if not result.isReady(api):
- release(result)
- result = await fetchToken()
- log "added new token to pool"
- tokenPool.add result
-
- if not result.isNil:
- inc result.pending
- else:
- raise rateLimitError()
-
-proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
- # avoid undefined behavior in race conditions
- if api in token.apis:
- let limit = token.apis[api]
- if limit.reset >= reset and limit.remaining < remaining:
- return
-
- token.apis[api] = RateLimit(remaining: remaining, reset: reset)
-
-proc poolTokens*(amount: int) {.async.} =
- var futs: seq[Future[Token]]
- for i in 0 ..< amount:
- futs.add fetchToken()
-
- for token in futs:
- var newToken: Token
-
- try: newToken = await token
- except: discard
-
- if not newToken.isNil:
- log "added new token to pool"
- tokenPool.add newToken
-
-proc initTokenPool*(cfg: Config) {.async.} =
- enableLogging = cfg.enableDebug
-
- while true:
- if tokenPool.countIt(not it.isLimited(Api.userTweets)) < cfg.minTokens:
- await poolTokens(min(4, cfg.minTokens - tokenPool.len))
- await sleepAsync(2000)
\ No newline at end of file
diff --git a/src/types.nim b/src/types.nim
index 38df172..34e61e8 100644
--- a/src/types.nim
+++ b/src/types.nim
@@ -12,25 +12,13 @@ type
TimelineKind* {.pure.} = enum
tweets, replies, media
- Api* {.pure.} = enum
- tweetDetail
- tweetResult
- photoRail
- search
- list
- listBySlug
- listMembers
- listTweets
- userRestId
- userScreenName
- favorites
- userTweets
- userTweetsAndReplies
- userMedia
- favoriters
- retweeters
- following
- followers
+ ApiUrl* = object
+ endpoint*: string
+ params*: seq[(string, string)]
+
+ ApiReq* = object
+ oauth*: ApiUrl
+ cookie*: ApiUrl
RateLimit* = object
remaining*: int
@@ -43,14 +31,14 @@ type
init*: Time
lastUse*: Time
pending*: int
- apis*: Table[Api, RateLimit]
+ apis*: Table[string, RateLimit]
GuestAccount* = ref object
id*: int64
oauthToken*: string
oauthSecret*: string
pending*: int
- apis*: Table[Api, RateLimit]
+ apis*: Table[string, RateLimit]
Error* = enum
null = 0
@@ -308,6 +296,7 @@ type
enableDebug*: bool
proxy*: string
proxyAuth*: string
+ disableTid*: bool
cookieHeader*: string
xCsrfToken*: string
diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim
index f1bbb80..7b29bb2 100644
--- a/src/views/renderutils.nim
+++ b/src/views/renderutils.nim
@@ -26,7 +26,9 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
template verifiedIcon*(user: User): untyped {.dirty.} =
if user.verifiedType != VerifiedType.none:
let lower = ($user.verifiedType).toLowerAscii()
- icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account")
+ buildHtml(tdiv(class=(&"verified-icon {lower}"))):
+ icon "circle", class="verified-icon-circle", title=(&"Verified {lower} account")
+ icon "ok", class="verified-icon-check", title=(&"Verified {lower} account")
else:
text ""
diff --git a/src/views/rss.nimf b/src/views/rss.nimf
index cdfcd63..27c2f4e 100644
--- a/src/views/rss.nimf
+++ b/src/views/rss.nimf
@@ -2,6 +2,9 @@
## SPDX-License-Identifier: AGPL-3.0-only
#import strutils, xmltree, strformat, options, unicode
#import ../types, ../utils, ../formatters, ../prefs
+## Snowflake ID cutoff for RSS GUID format transition
+## Corresponds to approximately December 14, 2025 UTC
+#const guidCutoff = 2000000000000000000'i64
#
#proc getTitle(tweet: Tweet; retweet: string): string =
#if tweet.pinned: result = "Pinned: "
@@ -25,7 +28,25 @@
#end proc
#
#proc getDescription(desc: string; cfg: Config): string =
-Twitter feed for: ${desc}. Generated by ${cfg.hostname}
+Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
+#end proc
+#
+#proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
+#result = profile.tweets.content
+#if profile.pinned.isSome and result.len > 0:
+# let pinnedTweet = profile.pinned.get
+# var inserted = false
+# for threadIdx in 0 ..< result.len:
+# if not inserted:
+# for tweetIdx in 0 ..< result[threadIdx].len:
+# if result[threadIdx][tweetIdx].id < pinnedTweet.id:
+# result[threadIdx].insert(pinnedTweet, tweetIdx)
+# inserted = true
+# end if
+# end for
+# end if
+# end for
+#end if
#end proc
#
#proc renderRssTweet(tweet: Tweet; cfg: Config): string =
@@ -38,23 +59,34 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
})
# end for
#elif tweet.video.isSome:
-.thumb)})
+
+
Video
+
+
#elif tweet.gif.isSome:
# let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
# let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
-##elif tweet.card.isSome:
-## let card = tweet.card.get()
-## if card.image.len > 0:
-##})
-## end if
+#elif tweet.card.isSome:
+# let card = tweet.card.get()
+# if card.image.len > 0:
+
+# end if
#end if
#if tweet.quote.isSome and get(tweet.quote).available:
-# let quoteLink = getLink(get(tweet.quote))
+# let quoteTweet = get(tweet.quote)
+# let quoteLink = urlPrefix & getLink(quoteTweet)
-Quoting: ${cfg.hostname}${quoteLink}
-${renderRssTweet(get(tweet.quote), cfg)}
+
+${quoteTweet.user.fullname} (@${quoteTweet.user.username})
+
+${renderRssTweet(quoteTweet, cfg)}
+
+
+
#end if
#end proc
#
@@ -72,12 +104,17 @@ ${renderRssTweet(get(tweet.quote), cfg)}
# if link in links: continue
# end if
# links.add link
+# let useGlobalGuid = parseBiggestInt(tweet.id) >= guidCutoff
-
${getTitle(tweet, retweet)}
@${tweet.user.username}
${getRfc822Time(tweet)}
+#if useGlobalGuid:
+ ${tweet.id}
+#else:
${urlPrefix & link}
+#end if
${urlPrefix & link}
# end for
@@ -108,8 +145,9 @@ ${renderRssTweet(get(tweet.quote), cfg)}
128
128
-#if profile.tweets.content.len > 0:
-${renderRssTweets(profile.tweets.content, cfg, userId=profile.user.id)}
+#let tweetsList = getTweetsWithPinned(profile)
+#if tweetsList.len > 0:
+${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
#end if
diff --git a/src/views/status.nim b/src/views/status.nim
index c3adb3d..090604e 100644
--- a/src/views/status.nim
+++ b/src/views/status.nim
@@ -28,14 +28,19 @@ proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode =
if thread.hasMore:
renderMoreReplies(thread)
-proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode =
+proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string; tweet: Tweet = nil): VNode =
buildHtml(tdiv(class="replies", id="r")):
+ var hasReplies = false
+ var replyCount = 0
for thread in replies.content:
if thread.content.len == 0: continue
+ hasReplies = true
+ replyCount += thread.content.len
renderReplyThread(thread, prefs, path)
- if replies.bottom.len > 0:
- renderMore(Query(), replies.bottom, focus="#r")
+ if hasReplies and replies.bottom.len > 0:
+ if tweet == nil or not replies.beginning or replyCount < tweet.stats.replies:
+ renderMore(Query(), replies.bottom, focus="#r")
proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode =
let hasAfter = conv.after.content.len > 0
@@ -70,6 +75,6 @@ proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode
if not conv.replies.beginning:
renderNewer(Query(), getLink(conv.tweet), focus="#r")
if conv.replies.content.len > 0 or conv.replies.bottom.len > 0:
- renderReplies(conv.replies, prefs, path)
+ renderReplies(conv.replies, prefs, path, conv.tweet)
renderToTop(focus="#m")