bunch of upstream changes

nitter.example.conf

@@ -26,12 +26,7 @@ enableRSS = true  # set this to false to disable RSS feeds
 enableDebug = false  # enable request logs and debug endpoints (/.accounts)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
-tokenCount = 10
+disableTid = false # enable this if cookie-based auth is failing
-# minimum amount of usable tokens. tokens are used to authorize API requests,
-# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
-# the limits reset every 15 minutes, and the pool is filled up so there's
-# always at least `tokenCount` usable tokens. only increase this if you receive
-# major bursts all the time and don't have a rate limiting setup via e.g. nginx
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]

src/api.nim

@@ -1,78 +1,114 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
+import asyncdispatch, httpclient, strutils, sequtils, sugar
 import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
 
+# Helper to generate params object for GraphQL requests
+proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
+  result.add ("variables", variables)
+  result.add ("features", gqlFeatures)
+  if fieldToggles.len > 0:
+    result.add ("fieldToggles", fieldToggles)
+
+proc apiUrl*(endpoint, variables: string; fieldToggles = ""): ApiUrl =
+  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
+
+proc apiReq*(endpoint, variables: string; fieldToggles = ""): ApiReq =
+  let url = apiUrl(endpoint, variables, fieldToggles)
+  return ApiReq(cookie: url, oauth: url)
+
+proc mediaUrl*(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor], """{"withArticlePlainText":false}"""),
+    oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor])
+  )
+
+proc userTweetsUrl*(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor])
+  )
+
+proc userTweetsAndRepliesUrl*(id: string; cursor: string): ApiReq =
+  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
+  result = ApiReq(
+    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor])
+  )
+
+proc tweetDetailUrl*(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    cookie: apiUrl(graphTweetDetail, tweetDetailVars % [id, cursor], tweetDetailFieldToggles),
+    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+  )
+
+proc userUrl*(username: string): ApiReq =
+  let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
+  result = ApiReq(
+    cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
+    oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
+  )
+
 proc getGraphUser*(username: string): Future[User] {.async.} =
   if username.len == 0: return
 
-  let
+  let headers = newHttpHeaders()
-    headers = newHttpHeaders()
   headers.add("Referer", """https://x.com/$1""" % username)
 
-  let
+  let js = await fetchRaw(userUrl(username), headers)
-    variables = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
-    fieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
-    params = {"variables": variables, "features": userFeatures, "fieldToggles": fieldToggles}
-    js = await fetch(graphUser ? params, Api.userScreenName, headers)
   result = parseGraphUser(js)
 
 proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
-  let
+  let headers = newHttpHeaders()
-    headers = newHttpHeaders()
   headers.add("Referer", """https://x.com/i/user/$1""" % id)
 
   let
-    variables = """{"userId":"$1"}""" % id
+    url = apiReq(graphUserById, """{"userId":"$1"}""" % id)
-    params = {"variables": variables, "features": userFeatures}
+    js = await fetchRaw(url, headers)
-    js = await fetch(graphUserById ? params, Api.userRestId, headers)
   result = parseGraphUser(js)
 
 proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
   if id.len == 0: return
 
-  let endpoint = case kind
-                 of TimelineKind.tweets: ""
-                 of TimelineKind.replies: "/with_replies"
-                 of TimelineKind.media: "/media"
   let
+    endpoint = case kind
+      of TimelineKind.tweets: ""
+      of TimelineKind.replies: "/with_replies"
+      of TimelineKind.media: "/media"
     headers = newHttpHeaders()
   headers.add("Referer", """https://x.com/$1$2""" % [id, endpoint])
 
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = if kind == TimelineKind.media: userMediaVariables % [id, cursor] else: userTweetsVariables % [id, cursor]
+    url = case kind
-    fieldToggles = """{"withArticlePlainText":true}"""
+      of TimelineKind.tweets: userTweetsUrl(id, cursor)
-    params = {"variables": variables, "features": gqlFeatures, "fieldToggles": fieldToggles}
+      of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
-    (url, apiId) = case kind
+      of TimelineKind.media: mediaUrl(id, cursor)
-                   of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
+    js = await fetch(url, headers)
-                   of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
-                   of TimelineKind.media: (graphUserMedia, Api.userMedia)
-    js = await fetch(url ? params, apiId, headers)
   result = parseGraphTimeline(js, if kind == TimelineKind.media: "" else: "user", after)
 
 proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = listTweetsVariables % [id, cursor]
+    url = apiReq(graphListTweets, restIdVars % [id, cursor])
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(url)
-    js = await fetch(graphListTweets ? params, Api.listTweets)
   result = parseGraphTimeline(js, "list", after).tweets
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
     variables = %*{"screenName": name, "listSlug": list}
-    params = {"variables": $variables, "features": gqlFeatures}
+    url = apiReq(graphListBySlug, $variables)
-  result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
+    js = await fetch(url)
+  result = parseGraphList(js)
 
 proc getGraphList*(id: string): Future[List] {.async.} =
   let 
-    variables = """{"listId":"$1"}""" % id
+    url = apiReq(graphListById, """{"listId": "$1"}""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(url)
-  result = parseGraphList(await fetch(graphListById ? params, Api.list))
+  result = parseGraphList(js)
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
   if list.id.len == 0: return
@@ -86,85 +122,45 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
-  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
-
-proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} =
-  if id.len == 0: return
-  var
-    variables = %*{
-      "userId": id,
-      "includePromotedContent":false,
-      "withClientEventToken":false,
-      "withBirdwatchNotes":true,
-      "withVoice":true,
-      "withV2Timeline":false
-    }
-  if after.len > 0:
-    variables["cursor"] = % after
   let 
-    url = consts.favorites ? {"variables": $variables, "features": gqlFeatures}
+    url = apiReq(graphListMembers, $variables)
-  result = parseGraphTimeline(await fetch(url, Api.favorites), after)
+    js = await fetchRaw(url)
+  result = parseGraphListMembers(js, after)
 
 proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
-  let
+
-    headers = newHttpHeaders()
+  let headers = newHttpHeaders()
   headers.add("Referer", """https://x.com/i/status/$1""" % id)
 
   let
-    variables = """{"rest_id":"$1"}""" % id
+    url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(url, headers)
-    js = await fetch(graphTweetResult ? params, Api.tweetResult, headers)
   result = parseGraphTweetResult(js)
 
 proc getGraphTweet*(id: string; after=""): Future[Conversation] {.async.} =
   if id.len == 0: return
-  let
+
-    headers = newHttpHeaders()
+  let headers = newHttpHeaders()
   headers.add("Referer", """https://x.com/i/status/$1""" % id)
 
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = tweetVariables % [id, cursor]
+    js = await fetch(tweetDetailUrl(id, cursor), headers)
-    params = {"variables": variables, "features": gqlFeatures, "fieldToggles": tweetFieldToggles}
-    js = await fetch(graphTweet ? params, Api.tweetDetail, headers)
   result = parseGraphConversation(js, id)
 
-proc getGraphFavoriters*(id: string; after=""): Future[UsersTimeline] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = reactorsVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphFavoriters ? params, Api.favoriters)
-  result = parseGraphFavoritersTimeline(js, id)
-
-proc getGraphRetweeters*(id: string; after=""): Future[UsersTimeline] {.async.} =
-  if id.len == 0: return
-  let
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = reactorsVariables % [id, cursor]
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphRetweeters ? params, Api.retweeters)
-  result = parseGraphRetweetersTimeline(js, id)
-
 proc getGraphFollowing*(id: string; after=""): Future[UsersTimeline] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = followVariables % [id, cursor]
+    js = await fetch(apiReq(graphFollowing, followVars % [id, cursor]))
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphFollowing ? params, Api.following)
   result = parseGraphFollowTimeline(js, id)
 
 proc getGraphFollowers*(id: string; after=""): Future[UsersTimeline] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    variables = followVariables % [id, cursor]
+    js = await fetch(apiReq(graphFollowers, followVars % [id, cursor]))
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphFollowers ? params, Api.followers)
   result = parseGraphFollowTimeline(js, id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
@@ -191,8 +187,10 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  let
-  result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[Tweets](js, after)
   result.query = query
 
 proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
@@ -211,17 +209,16 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
     variables["cursor"] = % after
     result.beginning = false
 
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  let
-  result = parseGraphSearch[User](await fetch(url, Api.search), after)
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[User](js, after)
   result.query = query
 
-proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
+proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
-  if name.len == 0: return
+  if id.len == 0: return
-  let
+  let js = await fetch(mediaUrl(id, ""))
-    ps = genParams({"screen_name": name, "trim_user": "true"},
+  result = parseGraphPhotoRail(js)
-                    count="18", ext=false)
-    url = photoRail ? ps
-  result = parsePhotoRail(await fetch(url, Api.photoRail))
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)

src/apiutils.nim

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import httpclient, asyncdispatch, options, strutils, uri, times, tables, math
 import jsony, packedjson, zippy
-import types, tokens, consts, parserutils, http_pool
+import types, consts, parserutils, http_pool, tid
 import experimental/types/common
 import config
 
@@ -9,42 +9,34 @@ const
   rlRemaining = "x-rate-limit-remaining"
   rlReset = "x-rate-limit-reset"
 
-var pool: HttpPool
+var
+  pool: HttpPool
+  disableTid: bool
 
-proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
+proc setDisableTid*(disable: bool) =
-                count="20"; ext=true): seq[(string, string)] =
+  disableTid = disable
-  result = timelineParams
+
-  for p in pars:
+proc toUrl(req: ApiReq): Uri =
-    result &= p
+  let c = req.cookie
-  if ext:
+  parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
-    result &= ("include_ext_alt_text", "1")
+
-    result &= ("include_ext_media_stats", "1")
+proc rateLimitError*(): ref RateLimitError =
-    result &= ("include_ext_media_availability", "1")
+  newException(RateLimitError, "rate limited")
-  if count.len > 0:
-    result &= ("count", count)
-  if cursor.len > 0:
-    # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to turn them back into a plus
-    if " " in cursor:
-      result &= ("cursor", cursor.replace(" ", "+"))
-    else:
-      result &= ("cursor", cursor)
 
-#proc genHeaders*(token: Token = nil): HttpHeaders =
 proc genHeaders*(): HttpHeaders =
   let
     t = getTime()
     ffVersion = floor(124 + (t.toUnix() - 1710892800) / 2419200)
   result = newHttpHeaders({
     "Connection": "keep-alive",
-    "Authorization": auth,
+    "Authorization": bearerToken,
     "Content-Type": "application/json",
     "Accept-Encoding": "gzip",
     "Accept-Language": "en-US,en;q=0.5",
     "Accept": "*/*",
     "DNT": "1",
     "Host": "x.com",
-    "Origin": "https://x.com/",
+    "Origin": "https://x.com",
     "Sec-Fetch-Dest": "empty",
     "Sec-Fetch-Mode": "cors",
     "Sec-Fetch-Site": "same-origin",
@@ -56,30 +48,20 @@ proc genHeaders*(): HttpHeaders =
     "x-twitter-client-language": "en"
   }, true)
 
-#template updateToken() =
-#  if resp.headers.hasKey(rlRemaining):
-#    let
-#      remaining = parseInt(resp.headers[rlRemaining])
-#      reset = parseInt(resp.headers[rlReset])
-#    token.setRateLimit(api, remaining, reset)
-
 template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
   once:
     pool = HttpPool()
 
-  #var token = await getToken(api)
-  #if token.tok.len == 0:
-  #  raise rateLimitError()
-
   if len(cfg.cookieHeader) != 0:
     additional_headers.add("Cookie", cfg.cookieHeader)
+    if not disableTid:
+      additional_headers.add("x-client-transaction-id", await genTid(url.path))
 
   if len(cfg.xCsrfToken) != 0:
     additional_headers.add("x-csrf-token", cfg.xCsrfToken)
 
   try:
     var resp: AsyncResponse
-    #var headers = genHeaders(token)
     var headers = genHeaders()
     for key, value in additional_headers.pairs():
       headers.add(key, value)
@@ -102,7 +84,6 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
       let
         remaining = parseInt(resp.headers[rlRemaining])
         reset = parseInt(resp.headers[rlReset])
-      #token.setRateLimit(api, remaining, reset)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -111,39 +92,27 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
       if result.startsWith("{\"errors"):
         let errors = result.fromJson(Errors)
         if errors in {expiredToken, badToken, authorizationError}:
-          echo "fetch error: ", errors
-          #release(token, invalid=true)
           raise rateLimitError()
         elif errors in {rateLimited}:
-          # rate limit hit, resets after 24 hours
-          #setLimited(account, api)
           raise rateLimitError()
       elif result.startsWith("429 Too Many Requests"):
-        #echo "[accounts] 429 error, API: ", api, ", token: ", token[]
-        #account.apis[api].remaining = 0
-        # rate limit hit, resets after the 15 minute window
         raise rateLimitError()
 
     fetchBody
 
-    #release(token, used=true)
-
     if resp.status == $Http400:
-      let errText = "body: '" & result & "' url: " & $url
+      echo "ERROR 400, ", url.path, ": ", result
-      raise newException(InternalError, errText)
+      raise newException(InternalError, $url)
   except InternalError as e:
     raise e
   except BadClientError as e:
-    #release(token, used=true)
     raise e
   except OSError as e:
     raise e
   except ProtocolError as e:
     raise e
   except Exception as e:
-    echo "error: ", e.name, ", msg: ", e.msg, ", url: ", url
+    echo "error: ", e.name, ", msg: ", e.msg, ", url: ", url.path
-    #if "length" notin e.msg and "descriptor" notin e.msg:
-      #release(token, invalid=true)
     raise rateLimitError()
 
 template retry(bod) =
@@ -152,36 +121,26 @@ template retry(bod) =
   except ProtocolError:
     bod
 
-proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
+proc fetch*(req: ApiReq; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
   retry:
     var body: string
+    let url = req.toUrl()
     fetchImpl(body, additional_headers):
       if body.startsWith('{') or body.startsWith('['):
         result = parseJson(body)
       else:
-        echo resp.status, ": ", body, " --- url: ", url
+        echo resp.status, " - non-json for: ", url, ", body: ", result
         result = newJNull()
 
-      #updateToken()
-
       let error = result.getError
       if error in {expiredToken, badToken}:
-        echo "fetch error: ", result.getError
+        echo "Fetch error, API: ", url.path, ", error: ", result.getError
-        #release(token, invalid=true)
         raise rateLimitError()
 
-proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
+proc fetchRaw*(req: ApiReq; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
   retry:
+    let url = req.toUrl()
     fetchImpl(result, additional_headers):
       if not (result.startsWith('{') or result.startsWith('[')):
-        echo resp.status, ": ", result, " --- url: ", url
+        echo resp.status, " - non-json for: ", url, ", body: ", result
         result.setLen(0)
-
-      #updateToken()
-
-      if result.startsWith("{\"errors"):
-        let errors = result.fromJson(Errors)
-        if errors in {expiredToken, badToken}:
-          echo "fetch error: ", errors
-          #release(token, invalid=true)
-          raise rateLimitError()

src/auth.nim

@@ -126,7 +126,7 @@ proc getAccountPoolDebug*(): JsonNode =
 proc rateLimitError*(): ref RateLimitError =
   newException(RateLimitError, "rate limited")
 
-proc isLimited(account: GuestAccount; api: Api): bool =
+proc isLimited(account: GuestAccount; req: ApiReq): bool =
   if account.isNil:
     return true
 
@@ -157,9 +157,9 @@ proc release*(account: GuestAccount) =
   if account.isNil: return
   dec account.pending
 
-proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
+proc getGuestAccount*(req: ApiReq): Future[GuestAccount] {.async.} =
   for i in 0 ..< accountPool.len:
-    if result.isReady(api): break
+    if result.isReady(req): break
     result = accountPool.sample()
 
   if not result.isNil and result.isReady(api):

src/config.nim

@@ -42,6 +42,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
     enableDebug: cfg.get("Config", "enableDebug", false),
     proxy: cfg.get("Config", "proxy", ""),
     proxyAuth: cfg.get("Config", "proxyAuth", ""),
+    disableTid: cfg.get("Config", "disableTid", false),
     cookieHeader: cfg.get("Config", "cookieHeader", ""),
     xCsrfToken: cfg.get("Config", "xCsrfToken", "")
   )

src/consts.nim

@@ -1,54 +1,30 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils, strutils
+import strutils
 
 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
-
   consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
   consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
+  bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 
-  api = parseUri("https://x.com/i/api")
+  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
-  activate* = $(api / "1.1/guest/activate.json")
+  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
-
+  graphUserById* = "Bbaot8ySMtJD7K2t01gW7A/UserByRestId"
-  photoRail* = api / "1.1/statuses/media_timeline.json"
+  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
-
+  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
-  timelineApi = api / "2/timeline"
+  graphUserTweets* = "lZRf8IC-GTuGxDwcsHW8aw/UserTweets"
-
+  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
-  graphql = api / "graphql"
+  graphUserMedia* = "1D04dx9H2pseMQAbMjXTvQ/UserMedia"
-  graphUser* = graphql / "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
+  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphUserById* = graphql / "Bbaot8ySMtJD7K2t01gW7A/UserByRestId"
+  graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
-  graphUserTweets* = graphql / "lZRf8IC-GTuGxDwcsHW8aw/UserTweets"
+  graphTweetDetail* = "6QzqakNMdh_YzBAR9SYPkQ/TweetDetail"
-  graphUserTweetsAndReplies* = graphql / "gXCeOBFsTOuimuCl1qXimg/UserTweetsAndReplies"
+  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
-  graphUserMedia* = graphql / "1D04dx9H2pseMQAbMjXTvQ/UserMedia"
+  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
-  graphTweet* = graphql / "6QzqakNMdh_YzBAR9SYPkQ/TweetDetail"
+  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
-  graphTweetResult* = graphql / "kLXoXTloWpv9d2FSXRg-Tg/TweetResultByRestId"
+  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
-  graphTweetHistory* = graphql / "WT7HhrzWulh4yudKJaR10Q/TweetEditHistory"
+  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
-  graphSearchTimeline* = graphql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
+  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
-  graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
+  graphFollowers* = "Efm7xwLreAw77q2Fq7rX-Q/Followers"
-  graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
+  graphFollowing* = "e0UtTAwQqgLKBllQxMgVxQ/Following"
-  graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
-  graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
-  graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters"
-  graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters"
-  graphFollowers* = graphql / "SCu9fVIlCUm-BM8-tL5pkQ/Followers"
-  graphFollowing* = graphql / "S5xUN9s2v4xk50KWGGvyvQ/Following"
-  favorites* = graphql / "eSSNbhECHHWWALkkQq-YTA/Likes"
-
-  timelineParams* = {
-    "include_can_media_tag": "1",
-    "include_cards": "1",
-    "include_entities": "1",
-    "include_profile_interstitial_type": "0",
-    "include_quote_count": "0",
-    "include_reply_count": "0",
-    "include_user_entities": "0",
-    "include_ext_reply_count": "0",
-    "include_ext_media_color": "0",
-    "cards_platform": "Web-13",
-    "tweet_mode": "extended",
-    "send_error_codes": "1",
-    "simple_quoted_tweet": "1"
-  }.toSeq
 
   gqlFeatures* = """{
   "rweb_video_screen_enabled": false,
@@ -84,85 +60,77 @@ const
   "responsive_web_grok_image_annotation_enabled": true,
   "responsive_web_grok_imagine_annotation_enabled": true,
   "responsive_web_grok_community_note_auto_translation_is_enabled": false,
-  "responsive_web_enhance_cards_enabled": false
+  "responsive_web_enhance_cards_enabled": false,
+  "payments_enabled": false,
+  "responsive_web_twitter_article_notes_tab_enabled": false,
+  "hidden_profile_subscriptions_enabled": false,
+  "subscriptions_verification_info_verified_since_enabled": false,
+  "subscriptions_verification_info_is_identity_verified_enabled": false,
+  "highlights_tweets_tab_ui_enabled": false,
+  "subscriptions_feature_can_gift_premium": false
 }""".replace(" ", "").replace("\n", "")
 
-  userFeatures* = """{
+  tweetVars* = """{
-  "hidden_profile_subscriptions_enabled": true,
+  "postId": "$1",
-  "profile_label_improvements_pcf_label_in_post_enabled": true,
+  $2
-  "responsive_web_profile_redirect_enabled": false,
+  "includeHasBirdwatchNotes": false,
-  "rweb_tipjar_consumption_enabled": true,
+  "includePromotedContent": false,
-  "verified_phone_label_enabled": false,
+  "withBirdwatchNotes": false,
-  "subscriptions_verification_info_is_identity_verified_enabled": true,
+  "withVoice": false,
-  "subscriptions_verification_info_verified_since_enabled": true,
+  "withV2Timeline": true
-  "highlights_tweets_tab_ui_enabled": true,
-  "responsive_web_twitter_article_notes_tab_enabled": true,
-  "subscriptions_feature_can_gift_premium": true,
-  "creator_subscriptions_tweet_preview_api_enabled": true,
-  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
-  "responsive_web_graphql_timeline_navigation_enabled": true
 }""".replace(" ", "").replace("\n", "")
 
-  tweetVariables* = """{
+  tweetDetailVars* = """{
   "focalTweetId": "$1",
   $2
+  "referrer": "profile",
   "with_rux_injections": false,
   "rankingMode": "Relevance",
-  "includePromotedContent": false,
+  "includePromotedContent": true,
   "withCommunity": true,
-  "withQuickPromoteEligibilityTweetFields": false,
+  "withQuickPromoteEligibilityTweetFields": true,
   "withBirdwatchNotes": true,
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
 
-  tweetFieldToggles* = """{
+  restIdVars* = """{
-  "withArticleRichContentState": true,
+  "rest_id": "$1", $2
-  "withArticlePlainText": false,
+  "count": 20
-  "withGrokAnalyze": false,
+}"""
-  "withDisallowedReplyControls": false
+
+  userMediaVars* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withClientEventToken": false,
+  "withBirdwatchNotes": false,
+  "withVoice": true
 }""".replace(" ", "").replace("\n", "")
 
-  userTweetsVariables* = """{
+  userTweetsVars* = """{
-  "userId": "$1",
+  "userId": "$1", $2
-  $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withQuickPromoteEligibilityTweetFields": true,
+  "withVoice": true
+}""".replace(" ", "").replace("\n", "")
+
+  userTweetsAndRepliesVars* = """{
+  "userId": "$1", $2
   "count": 20,
   "includePromotedContent": false,
   "withCommunity": true,
   "withVoice": true
 }""".replace(" ", "").replace("\n", "")
 
-  listTweetsVariables* = """{
+  followVars* = """{
-  "rest_id": "$1",
+  "userId": "$1",
-  $2
-  "count": 20
-}""".replace(" ", "").replace("\n", "")
-
-  reactorsVariables* = """{
-  "tweetId": "$1",
   $2
   "count": 20,
   "includePromotedContent": false
 }""".replace(" ", "").replace("\n", "")
 
-  followVariables* = """{
-  "userId": "$1",
-  $2
-  "count": 20,
-  "includePromotedContent": false,
-  "withGrokTranslatedBio": false
-}""".replace(" ", "").replace("\n", "")
 
-  userMediaVariables* = """{
+  userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
-  "userId": "$1",
+  userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
-  $2
+  tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""
-  "count": 20,
-  "includePromotedContent": false,
-  "withClientEventToken": false,
-  "withBirdwatchNotes": true,
-  "withVoice": true
-}""".replace(" ", "").replace("\n", "")
-
-  tweetHistoryVariables* = """{
-  "tweetId": "$1",
-  "withQuickPromoteEligibilityTweetFields": false
-}""".replace(" ", "").replace("\n", "")

src/experimental/parser/graphql.nim

@@ -1,21 +1,53 @@
-import options
+import options, strutils
 import jsony
-import user, ../types/[graphuser, graphlistmembers]
+import user, utils, ../types/[graphuser, graphlistmembers]
 from ../../types import User, VerifiedType, Result, Query, QueryKind
 
-#proc parseGraphUser*(json: string): User =
+proc parseUserResult*(userResult: UserResult): User =
-#  if json.len == 0 or json[0] != '{':
+  result = userResult.legacy
-#    return
+
-#
+  if result.verifiedType == none and userResult.isBlueVerified:
-#  let raw = json.fromJson(GraphUser)
+    result.verifiedType = blue
-#
+
-#  if raw.data.userResult.result.unavailableReason.get("") == "Suspended":
+  if result.username.len == 0 and userResult.core.screenName.len > 0:
-#    return User(suspended: true)
+    result.id = userResult.restId
-#
+    result.username = userResult.core.screenName
-#  result = raw.data.userResult.result.legacy
+    result.fullname = userResult.core.name
-#  result.id = raw.data.userResult.result.restId
+    result.userPic = userResult.avatar.imageUrl.replace("_normal", "")
-#  if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified:
+
-#    result.verifiedType = blue
+    if userResult.privacy.isSome:
+      result.protected = userResult.privacy.get.protected
+
+    if userResult.location.isSome:
+      result.location = userResult.location.get.location
+
+    if userResult.core.createdAt.len > 0:
+      result.joinDate = parseTwitterDate(userResult.core.createdAt)
+
+    if userResult.verification.isSome:
+      let v = userResult.verification.get
+      if v.verifiedType != VerifiedType.none:
+        result.verifiedType = v.verifiedType
+
+    if userResult.profileBio.isSome and result.bio.len == 0:
+      result.bio = userResult.profileBio.get.description
+
+proc parseGraphUser*(json: string): User =
+  if json.len == 0 or json[0] != '{':
+    return
+
+  let 
+    raw = json.fromJson(GraphUser)
+    userResult = 
+      if raw.data.userResult.isSome: raw.data.userResult.get.result
+      elif raw.data.user.isSome: raw.data.user.get.result
+      else: UserResult()
+
+  if userResult.unavailableReason.get("") == "Suspended" or
+     userResult.reason.get("") == "Suspended":
+    return User(suspended: true)
+
+  result = parseUserResult(userResult)
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
   result = Result[User](
@@ -31,7 +63,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
         of TimelineTimelineItem:
           let userResult = entry.content.itemContent.userResults.result
           if userResult.restId.len > 0:
-            result.content.add userResult.legacy
+            result.content.add parseUserResult(userResult)
         of TimelineTimelineCursor:
           if entry.content.cursorType == "Bottom":
             result.bottom = entry.content.value

src/experimental/parser/tid.nim

@@ -0,0 +1,8 @@
+import jsony
+import ../types/tid
+export TidPair
+
+proc parseTidPairs*(raw: string): seq[TidPair] =
+  result = raw.fromJson(seq[TidPair])
+  if result.len == 0:
+    raise newException(ValueError, "Parsing pairs failed: " & raw)

src/experimental/types/graphuser.nim

@@ -1,15 +1,48 @@
-import options
+import options, strutils
-from ../../types import User
+from ../../types import User, VerifiedType
 
 type
   GraphUser* = object
-    data*: tuple[userResult: UserData]
+    data*: tuple[userResult: Option[UserData], user: Option[UserData]]
 
   UserData* = object
     result*: UserResult
 
-  UserResult = object
+  UserCore* = object
+    name*: string
+    screenName*: string
+    createdAt*: string
+
+  UserBio* = object
+    description*: string
+
+  UserAvatar* = object
+    imageUrl*: string
+
+  Verification* = object
+    verifiedType*: VerifiedType
+
+  Location* = object
+    location*: string
+
+  Privacy* = object
+    protected*: bool
+
+  UserResult* = object
     legacy*: User
     restId*: string
     isBlueVerified*: bool
+    core*: UserCore
+    avatar*: UserAvatar
     unavailableReason*: Option[string]
+    reason*: Option[string]
+    privacy*: Option[Privacy]
+    profileBio*: Option[UserBio]
+    verification*: Option[Verification]
+    location*: Option[Location]
+
+proc enumHook*(s: string; v: var VerifiedType) =
+  v = try:
+    parseEnum[VerifiedType](s)
+  except:
+    VerifiedType.none

src/experimental/types/tid.nim

@@ -0,0 +1,4 @@
+type
+  TidPair* = object
+    animationKey*: string
+    verification*: string

src/nitter.nim

@@ -6,7 +6,7 @@ from htmlgen import a
 
 import jester
 
-import types, config, prefs, formatters, redis_cache, http_pool
+import types, config, prefs, formatters, redis_cache, http_pool, apiutils
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, list, rss, #debug,
@@ -34,6 +34,7 @@ setHmacKey(cfg.hmacKey)
 setProxyEncoding(cfg.base64Media)
 setMaxHttpConns(cfg.httpMaxConns)
 setHttpProxy(cfg.proxy, cfg.proxyAuth)
+setDisableTid(cfg.disableTid)
 initAboutPage(cfg.staticDir)
 
 waitFor initRedisPool(cfg)
@@ -104,7 +105,6 @@ routes:
   extend preferences, ""
   extend resolver, ""
   extend embed, ""
-  #extend debug, ""
   extend activityspoof, ""
   extend api, ""
   extend unsupported, ""

src/parser.nim

@@ -45,6 +45,7 @@ proc parseGraphUser*(js: JsonNode): User =
   result.fullname = user{"core", "name"}.getStr
   result.joinDate = parseTwitterDate(user{"core", "created_at"}.getStr)
   result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
+  result.protected = user{"privacy", "protected"}.getBool
 
   let label = user{"affiliates_highlighted_label", "label"}
   if not label.isNull:
@@ -449,23 +450,6 @@ proc parseTimeline*(js: JsonNode; after=""): Profile =
         else:
           result.tweets.top = cursor{"value"}.getStr
 
-proc parsePhotoRail*(js: JsonNode): PhotoRail =
-  with error, js{"error"}:
-    if error.getStr == "Not authorized.":
-      return
-
-  for tweet in js:
-    let
-      t = parseTweet(tweet, js{"tweet_card"})
-      url = if t.photos.len > 0: t.photos[0].url
-            elif t.video.isSome: get(t.video).thumb
-            elif t.gif.isSome: get(t.gif).thumb
-            elif t.card.isSome: get(t.card).image
-            else: ""
-
-    if url.len == 0: continue
-    result.add GalleryPhoto(url: url, tweetId: $t.id)
-
 proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
   if js.kind == JNull:
     return Tweet()
@@ -531,6 +515,34 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
   else:
     result.stats.views = -1
 
+proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
+  result = @[]
+
+  let instructions =
+    ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
+
+  for i in instructions:
+    if i{"__typename"}.getStr == "TimelineAddEntries":
+      for e in i{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "content", "tweetResult", "result"}:
+            let t = parseGraphTweet(tweetResult, false)
+            if not t.available:
+              t.id = $parseBiggestInt(entryId.getId())
+
+            let url =
+              if t.photos.len > 0: t.photos[0].url
+              elif t.video.isSome: get(t.video).thumb
+              elif t.gif.isSome: get(t.gif).thumb
+              elif t.card.isSome: get(t.card).image
+              else: ""
+
+            result.add GalleryPhoto(url: url, tweetId: t.id)
+
+            if result.len == 16:
+              break
+
 proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
   for t in js{"content", "items"}:
     let entryId = t{"entryId"}.getStr

src/redis_cache.nim

@@ -87,7 +87,7 @@ proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: PhotoRail; name: string) {.async.} =
-  await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
+  await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
 
 proc cache*(data: User) {.async.} =
   if data.username.len == 0: return
@@ -145,6 +145,9 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
   elif fetch:
     result = await getGraphUser(username)
     await cache(result)
+    if result.id.len > 0:
+      await setEx("i:" & result.id, baseCacheTime, result.username)
+      await cacheUserId(result.username, result.id)
 
 proc getCachedUsername*(userId: string): Future[string] {.async.} =
   let
@@ -174,14 +177,14 @@ proc getCachedTweet*(id: string; after=""): Future[Conversation] {.async.} =
   if not result.isNil and after.len > 0:
     result.replies = await getReplies(id, after)
 
-proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
+proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
-  if name.len == 0: return
+  if id.len == 0: return
-  let rail = await get("pr:" & toLower(name))
+  let rail = await get("pr2:" & toLower(id))
   if rail != redisNil:
     rail.deserialize(PhotoRail)
   else:
-    result = await getPhotoRail(name)
+    result = await getPhotoRail(id)
-    await cache(result, name)
+    await cache(result, id)
 
 proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
   let list = if id.len == 0: redisNil

src/routes/activityspoof.nim

@@ -19,14 +19,15 @@ proc createActivityPubRouter*(cfg: Config) =
 
     get "/api/v1/accounts/@id":
       let id = @"id"
-      if id.len > 19 or id.any(c => not c.isDigit):
+      #if id.len > 19 or id.any(c => not c.isDigit):
+      if not id.allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}):
         resp Http404, {"Content-Type": "application/json"}, """{"error":"Invalid account ID"}"""
 
-      var username = await getCachedUsername(id)
+      #var username = await getCachedUsername(id)
-      if username.len == 0:
+      #if username.len == 0:
-        resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
+        #resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
 
-      let user = await getCachedUser(username)
+      let user = await getCachedUser(id)
       if user.suspended or user.id.len == 0:
         resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
 
@@ -189,9 +190,8 @@ proc createActivityPubRouter*(cfg: Config) =
       postJson["edited_at"] = newJNull()
       postJson["reblog"] = newJNull()
       if tweet.replyId.len != 0:
-        let replyUser = await getCachedUser(tweet.replyHandle)
         postJson["in_reply_to_id"] = %(&"{tweet.replyId}")
-        postJson["in_reply_to_account_id"] = %replyUser.id
+        postJson["in_reply_to_account_id"] = %tweet.replyHandle
       else:
         postJson["in_reply_to_id"] = newJNull()
         postJson["in_reply_to_account_id"] = newJNull()

src/routes/debug.nim

@@ -1,17 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import jester
-import router_utils
-import ".."/[tokens, types]
-
-proc createDebugRouter*(cfg: Config) =
-  router debug:
-    get "/.tokens":
-      cond cfg.enableDebug
-      respJson getPoolJson()
-
-    #get "/.health":
-      #respJson getAccountPoolHealth()
-
-    #get "/.accounts":
-      #cond cfg.enableDebug
-      #respJson getAccountPoolDebug()

src/routes/status.nim

@@ -5,7 +5,7 @@ import jester, karax/vdom
 
 import router_utils
 import ".."/[types, formatters, api, redis_cache]
-import ../views/[general, status, search, mastoapi]
+import ../views/[general, status, mastoapi]
 
 export json, uri, sequtils, options, sugar, times
 export router_utils

src/routes/timeline.nim

@@ -48,7 +48,7 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
   let
     rail =
       skipIf(skipRail or query.kind == media, @[]):
-        getCachedPhotoRail(name)
+        getCachedPhotoRail(userId)
 
     user = getCachedUser(name)
 
@@ -111,6 +111,7 @@ proc createTimelineRouter*(cfg: Config) =
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
       cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
+      cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_', ','})
       cond @"tab" in ["with_replies", "media", "search", "following", "followers", ""]
       let
         prefs = cookiePrefs()

src/routes/twitter_api.nim

@@ -4,7 +4,7 @@ import json, asyncdispatch, options, uri
 import times
 import jester
 import router_utils
-import ".."/[types, api, apiutils, query, consts, redis_cache]
+import ".."/[types, api, apiutils, redis_cache]
 import httpclient, strutils
 import sequtils
 
@@ -81,60 +81,49 @@ proc getUserTweetsJson*(id: string): Future[JsonNode] {.async.} =
 
   result = response
 
-proc searchTimeline*(query: Query; after=""): Future[string] {.async.} =
-  let q = genQueryParam(query)
-  var
-    variables = %*{
-      "rawQuery": q,
-      "count": 20,
-      "product": "Latest",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
-    }
-  if after.len > 0:
-    variables["cursor"] = % after
-  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
-  result = await fetchRaw(url, Api.search)
-
 proc getUserTweets*(id: string; after=""): Future[string] {.async.} =
   if id.len == 0: return
-  let
+
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+  let headers = newHttpHeaders()
-    variables = userTweetsVariables % [id, cursor]
+  headers.add("Referer", """https://x.com/$1""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+
-  result = await fetchRaw(graphUserTweets ? params, Api.userTweets)
+  let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+  result = await fetchRaw(userTweetsUrl(id, cursor), headers)
 
 proc getUserReplies*(id: string; after=""): Future[string] {.async.} =
   if id.len == 0: return
-  let
+
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+  let headers = newHttpHeaders()
-    variables = userTweetsVariables % [id, cursor]
+  headers.add("Referer", """https://x.com/$1/with_replies""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+
-  result = await fetchRaw(graphUserTweets ? params, Api.userTweetsAndReplies)
+  let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+  result = await fetchRaw(userTweetsAndRepliesUrl(id, cursor), headers)
 
 proc getUserMedia*(id: string; after=""): Future[string] {.async.} =
   if id.len == 0: return
-  let
+
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+  let headers = newHttpHeaders()
-    variables = userMediaVariables % [id, cursor]
+  headers.add("Referer", """https://x.com/$1/media""" % id)
-    params = {"variables": variables, "features": gqlFeatures}
+
-  result = await fetchRaw(graphUserMedia ? params, Api.userMedia)
+  let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+  result = await fetchRaw(mediaUrl(id, cursor), headers)
 
 proc getTweetById*(id: string; after=""): Future[string] {.async.} =
-  let
+  if id.len == 0: return
-    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+
-    variables = tweetVariables % [id, cursor]
+  let headers = newHttpHeaders()
-    params = {"variables": variables, "features": gqlFeatures}
+  headers.add("Referer", """https://x.com/i/status/$1""" % id)
-  result = await fetchRaw(graphTweet ? params, Api.tweetDetail)
+
+  let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+  result = await fetchRaw(tweetDetailUrl(id, cursor), headers)
 
 proc getUser*(username: string): Future[string] {.async.} =
   if username.len == 0: return
-  let
+
-    variables = """{"screen_name":"$1"}""" % username
+  let headers = newHttpHeaders()
-    fieldToggles = """{"withAuxiliaryUserLabels":true}"""
+  headers.add("Referer", """https://x.com/$1""" % username)
-    params = {"variables": variables, "features": gqlFeatures, "fieldToggles": fieldToggles}
+
-  result = await fetchRaw(graphUser ? params, Api.userScreenName)
+  result = await fetchRaw(userUrl(username), headers)
 
 proc createTwitterApiRouter*(cfg: Config) =
   router api:
@@ -146,16 +135,11 @@ proc createTwitterApiRouter*(cfg: Config) =
       let response = await getUser(username)
       resp Http200, { "Content-Type": "application/json" }, response
 
-    #get "/api/user/@id/tweets":
+    #get "/api/user/@username/timeline":
-    #  let id = @"id"
+    #  let username = @"username"
-    #  let response = await getUserTweetsJson(id)
+    #  let query = Query(fromUser: @[username])
-    #  respJson response
+    #  let response = await searchTimeline(query)
-
+    #  resp Http200, { "Content-Type": "application/json" }, response
-    get "/api/user/@username/timeline":
-      let username = @"username"
-      let query = Query(fromUser: @[username])
-      let response = await searchTimeline(query)
-      resp Http200, { "Content-Type": "application/json" }, response
 
     get "/api/user/@id/tweets":
       let id = @"id"

src/sass/general.scss

@@ -23,6 +23,7 @@
     font-weight: bold;
     width: 30px;
     height: 30px;
+    padding: 0px 5px 1px 8px;
   }
 
   input {

src/sass/index.scss

@@ -1,180 +1,202 @@
-@import '_variables';
+@import "_variables";
 
-@import 'tweet/_base';
+@import "tweet/_base";
-@import 'profile/_base';
+@import "profile/_base";
-@import 'general';
+@import "general";
-@import 'navbar';
+@import "navbar";
-@import 'inputs';
+@import "inputs";
-@import 'timeline';
+@import "timeline";
-@import 'search';
+@import "search";
 
 body {
-    // colors
+  // colors
-    --bg_color: #{$bg_color};
+  --bg_color: #{$bg_color};
-    --fg_color: #{$fg_color};
+  --fg_color: #{$fg_color};
-    --fg_faded: #{$fg_faded};
+  --fg_faded: #{$fg_faded};
-    --fg_dark: #{$fg_dark};
+  --fg_dark: #{$fg_dark};
-    --fg_nav: #{$fg_nav};
+  --fg_nav: #{$fg_nav};
 
-    --bg_panel: #{$bg_panel};
+  --bg_panel: #{$bg_panel};
-    --bg_elements: #{$bg_elements};
+  --bg_elements: #{$bg_elements};
-    --bg_overlays: #{$bg_overlays};
+  --bg_overlays: #{$bg_overlays};
-    --bg_hover: #{$bg_hover};
+  --bg_hover: #{$bg_hover};
 
-    --grey: #{$grey};
+  --grey: #{$grey};
-    --dark_grey: #{$dark_grey};
+  --dark_grey: #{$dark_grey};
-    --darker_grey: #{$darker_grey};
+  --darker_grey: #{$darker_grey};
-    --darkest_grey: #{$darkest_grey};
+  --darkest_grey: #{$darkest_grey};
-    --border_grey: #{$border_grey};
+  --border_grey: #{$border_grey};
 
-    --accent: #{$accent};
+  --accent: #{$accent};
-    --accent_light: #{$accent_light};
+  --accent_light: #{$accent_light};
-    --accent_dark: #{$accent_dark};
+  --accent_dark: #{$accent_dark};
-    --accent_border: #{$accent_border};
+  --accent_border: #{$accent_border};
 
-    --play_button: #{$play_button};
+  --play_button: #{$play_button};
-    --play_button_hover: #{$play_button_hover};
+  --play_button_hover: #{$play_button_hover};
 
-    --more_replies_dots: #{$more_replies_dots};
+  --more_replies_dots: #{$more_replies_dots};
-    --error_red: #{$error_red};
+  --error_red: #{$error_red};
 
-    --verified_blue: #{$verified_blue};
+  --verified_blue: #{$verified_blue};
-    --verified_business: #{$verified_business};
+  --verified_business: #{$verified_business};
-    --verified_government: #{$verified_government};
+  --verified_government: #{$verified_government};
-    --icon_text: #{$icon_text};
+  --icon_text: #{$icon_text};
 
-    --tab: #{$fg_color};
+  --tab: #{$fg_color};
-    --tab_selected: #{$accent};
+  --tab_selected: #{$accent};
 
-    --profile_stat: #{$fg_color};
+  --profile_stat: #{$fg_color};
 
-    background-color: var(--bg_color);
+  background-color: var(--bg_color);
-    color: var(--fg_color);
+  color: var(--fg_color);
-    font-family: $font_0, $font_1, $font_2, $font_3;
+  font-family: $font_0, $font_1;
-    font-size: 14px;
+  font-size: 14px;
-    line-height: 1.3;
+  line-height: 1.3;
-    margin: 0;
+  margin: 0;
 }
 
 * {
-    outline: unset;
+  outline: unset;
-    margin: 0;
+  margin: 0;
-    text-decoration: none;
+  text-decoration: none;
 }
 
 h1 {
-    display: inline;
+  display: inline;
 }
 
-h2, h3 {
+h2,
-    font-weight: normal;
+h3 {
+  font-weight: normal;
 }
 
 p {
-    margin: 14px 0;
+  margin: 14px 0;
 }
 
 a {
-    color: var(--accent);
+  color: var(--accent);
 
-    &:hover {
+  &:hover {
-        text-decoration: underline;
+    text-decoration: underline;
-    }
+  }
 }
 
 fieldset {
-    border: 0;
+  border: 0;
-    padding: 0;
+  padding: 0;
-    margin-top: -0.6em;
+  margin-top: -0.6em;
 }
 
 legend {
-    width: 100%;
+  width: 100%;
-    padding: .6em 0 .3em 0;
+  padding: 0.6em 0 0.3em 0;
-    border: 0;
+  border: 0;
-    font-size: 16px;
+  font-size: 16px;
-    font-weight: 600;
+  font-weight: 600;
-    border-bottom: 1px solid var(--border_grey);
+  border-bottom: 1px solid var(--border_grey);
-    margin-bottom: 8px;
+  margin-bottom: 8px;
 }
 
 .preferences .note {
-    border-top: 1px solid var(--border_grey);
+  border-top: 1px solid var(--border_grey);
-    border-bottom: 1px solid var(--border_grey);
+  border-bottom: 1px solid var(--border_grey);
-    padding: 6px 0 8px 0;
+  padding: 6px 0 8px 0;
-    margin-bottom: 8px;
+  margin-bottom: 8px;
-    margin-top: 16px;
+  margin-top: 16px;
 }
 
 ul {
-    padding-left: 1.3em;
+  padding-left: 1.3em;
 }
 
 .container {
-    display: flex;
+  display: flex;
-    flex-wrap: wrap;
+  flex-wrap: wrap;
-    box-sizing: border-box;
+  box-sizing: border-box;
-    padding-top: 50px;
+  padding-top: 50px;
-    margin: auto;
+  margin: auto;
-    min-height: 100vh;
+  min-height: 100vh;
 }
 
 .icon-container {
-    display: inline;
+  display: inline;
 }
 
 .overlay-panel {
-    max-width: 600px;
+  max-width: 600px;
-    width: 100%;
+  width: 100%;
-    margin: 0 auto;
+  margin: 0 auto;
-    margin-top: 10px;
+  margin-top: 10px;
-    background-color: var(--bg_overlays);
+  background-color: var(--bg_overlays);
-    padding: 10px 15px;
+  padding: 10px 15px;
-    align-self: start;
+  align-self: start;
 
-    ul {
+  ul {
-        margin-bottom: 14px;
+    margin-bottom: 14px;
-    }
+  }
 
-    p {
+  p {
-        word-break: break-word;
+    word-break: break-word;
-    }
+  }
 }
 
 .verified-icon {
-    color: var(--icon_text);
+  display: inline-block;
-    border-radius: 50%;
+  width: 14px;
-    flex-shrink: 0;
+  height: 14px;
-    margin: 2px 0 3px 3px;
+  margin-left: 2px;
-    padding-top: 3px;
-    height: 11px;
-    width: 14px;
-    font-size: 8px;
-    display: inline-block;
-    text-align: center;
-    vertical-align: middle;
 
-    &.blue {
+  .verified-icon-circle {
-        background-color: var(--verified_blue);
+    position: absolute;
+    font-size: 15px;
+  }
+
+  .verified-icon-check {
+    position: absolute;
+    font-size: 9px;
+    margin: 5px 3px;
+  }
+
+  &.blue {
+    .verified-icon-circle {
+      color: var(--verified_blue);
     }
 
-    &.business {
+    .verified-icon-check {
-        color: var(--bg_panel);
+      color: var(--icon_text);
-        background-color: var(--verified_business);
+    }
+  }
+
+  &.business {
+    .verified-icon-circle {
+      color: var(--verified_business);
     }
 
-    &.government {
+    .verified-icon-check {
-        color: var(--bg_panel);
+      color: var(--bg_panel);
-        background-color: var(--verified_government);
     }
+  }
+
+  &.government {
+    .verified-icon-circle {
+      color: var(--verified_government);
+    }
+
+    .verified-icon-check {
+      color: var(--bg_panel);
+    }
+  }
 }
 
-@media(max-width: 600px) {
+@media (max-width: 600px) {
-    .preferences-container {
+  .preferences-container {
-        max-width: 95vw;
+    max-width: 95vw;
-    }
+  }
 
-    .nav-item, .nav-item .icon-container {
+  .nav-item,
-        font-size: 16px;
+  .nav-item .icon-container {
-    }
+    font-size: 16px;
+  }
 }

src/sass/navbar.scss

@@ -1,89 +1,87 @@
-@import '_variables';
+@import "_variables";
 
 nav {
-    display: flex;
+  display: flex;
-    align-items: center;
+  align-items: center;
-    position: fixed;
+  position: fixed;
-    background-color: var(--bg_overlays);
+  background-color: var(--bg_overlays);
-    box-shadow: 0 0 4px $shadow;
+  box-shadow: 0 0 4px $shadow;
-    padding: 0;
+  padding: 0;
-    width: 100%;
+  width: 100%;
-    height: 50px;
+  height: 50px;
-    z-index: 1000;
+  z-index: 1000;
-    font-size: 16px;
+  font-size: 16px;
 
-    a, .icon-button button {
+  a,
-        color: var(--fg_nav);
+  .icon-button button {
-    }
+    color: var(--fg_nav);
+  }
 }
 
 .inner-nav {
-    margin: auto;
+  margin: auto;
-    box-sizing: border-box;
+  box-sizing: border-box;
-    padding: 0 10px;
+  padding: 0 10px;
-    display: flex;
+  display: flex;
-    align-items: center;
+  align-items: center;
-    flex-basis: 920px;
+  flex-basis: 920px;
-    height: 50px;
+  height: 50px;
 }
 
 .site-name {
-    font-size: 15px;
+  font-size: 15px;
-    font-weight: 600;
+  font-weight: 600;
-    line-height: 1;
+  line-height: 1;
 
-    &:hover {
+  &:hover {
-        color: var(--accent_light);
+    color: var(--accent_light);
-        text-decoration: unset;
+    text-decoration: unset;
-    }
+  }
 }
 
 .site-logo {
-    display: block;
+  display: block;
-    width: 35px;
+  width: 35px;
-    height: 35px;
+  height: 35px;
 }
 
 .nav-item {
-    display: flex;
+  display: flex;
-    flex: 1;
+  flex: 1;
-    line-height: 50px;
+  line-height: 50px;
-    height: 50px;
+  height: 50px;
-    overflow: hidden;
+  overflow: hidden;
-    flex-wrap: wrap;
+  flex-wrap: wrap;
-    align-items: center;
+  align-items: center;
 
-    &.right {
+  &.right {
-        text-align: right;
+    text-align: right;
-        justify-content: flex-end;
+    justify-content: flex-end;
-    }
+  }
 
-    &.right a {
+  &.right a:hover {
-        padding-left: 4px;
+    color: var(--accent_light);
-
+    text-decoration: unset;
-        &:hover {
+  }
-            color: var(--accent_light);
-            text-decoration: unset;
-        }
-    }
 }
 
 .lp {
-    height: 14px;
+  height: 14px;
-    display: inline-block;
+  display: inline-block;
-    position: relative;
+  position: relative;
-    top: 2px;
+  top: 2px;
-    fill: var(--fg_nav);
+  fill: var(--fg_nav);
 
-    &:hover {
+  &:hover {
-        fill: var(--accent_light);
+    fill: var(--accent_light);
-    }
+  }
 }
 
-.icon-info:before {
+.icon-info {
-    margin: 0 -3px;
+  margin: 0 -3px;
 }
 
 .icon-cog {
-    font-size: 15px;
+  font-size: 15px;
+  padding-left: 0 !important;
 }

src/sass/search.scss

@@ -1,120 +1,121 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
 
 .search-title {
-    font-weight: bold;
+  font-weight: bold;
-    display: inline-block;
+  display: inline-block;
-    margin-top: 4px;
+  margin-top: 4px;
 }
 
 .search-field {
+  display: flex;
+  flex-wrap: wrap;
+
+  button {
+    margin: 0 2px 0 0;
+    padding: 0px 1px 1px 4px;
+    height: 23px;
     display: flex;
-    flex-wrap: wrap;
+    align-items: center;
+  }
 
-    button {
+  .pref-input {
-        margin: 0 2px 0 0;
+    margin: 0 4px 0 0;
-        height: 23px;
+    flex-grow: 1;
-        display: flex;
+    height: 23px;
-        align-items: center;
+  }
-    }
 
-    .pref-input {
+  input[type="text"] {
-        margin: 0 4px 0 0;
+    height: calc(100% - 4px);
-        flex-grow: 1;
+    width: calc(100% - 8px);
-        height: 23px;
+  }
-    }
 
-    input[type="text"] {
+  > label {
-        height: calc(100% - 4px);
+    display: inline;
-        width: calc(100% - 8px);
+    background-color: var(--bg_elements);
-    }
+    color: var(--fg_color);
+    border: 1px solid var(--accent_border);
+    padding: 1px 1px 2px 4px;
+    font-size: 14px;
+    cursor: pointer;
+    margin-bottom: 2px;
 
-    > label {
+    @include input-colors;
-        display: inline;
+  }
-        background-color: var(--bg_elements);
-        color: var(--fg_color);
-        border: 1px solid var(--accent_border);
-        padding: 1px 6px 2px 6px;
-        font-size: 14px;
-        cursor: pointer;
-        margin-bottom: 2px;
 
-        @include input-colors;
+  @include create-toggle(search-panel, 200px);
-    }
-
-    @include create-toggle(search-panel, 200px);
 }
 
 .search-panel {
-    width: 100%;
+  width: 100%;
-    max-height: 0;
+  max-height: 0;
-    overflow: hidden;
+  overflow: hidden;
-    transition: max-height 0.4s;
+  transition: max-height 0.4s;
 
-    flex-grow: 1;
+  flex-grow: 1;
-    font-weight: initial;
+  font-weight: initial;
-    text-align: left;
+  text-align: left;
 
-    > div {
+  > div {
-        line-height: 1.7em;
+    line-height: 1.7em;
-    }
+  }
 
-    .checkbox-container {
+  .checkbox-container {
-        display: inline;
+    display: inline;
-        padding-right: unset;
+    padding-right: unset;
-        margin-bottom: unset;
+    margin-bottom: 5px;
-        margin-left: 23px;
+    margin-left: 23px;
-    }
+  }
 
-    .checkbox {
+  .checkbox {
-        right: unset;
+    right: unset;
-        left: -22px;
+    left: -22px;
-    }
+  }
 
-    .checkbox-container .checkbox:after {
+  .checkbox-container .checkbox:after {
-        top: -4px;
+    top: -4px;
-    }
+  }
 }
 
 .search-row {
-    display: flex;
+  display: flex;
-    flex-wrap: wrap;
+  flex-wrap: wrap;
-    line-height: unset;
+  line-height: unset;
 
-    > div {
+  > div {
-        flex-grow: 1;
+    flex-grow: 1;
-        flex-shrink: 1;
+    flex-shrink: 1;
-    }
+  }
+
+  input {
+    height: 21px;
+  }
+
+  .pref-input {
+    display: block;
+    padding-bottom: 5px;
 
     input {
-        height: 21px;
+      height: 21px;
-    }
+      margin-top: 1px;
-
-    .pref-input {
-        display: block;
-        padding-bottom: 5px;
-
-        input {
-            height: 21px;
-            margin-top: 1px;
-        }
     }
+  }
 }
 
 .search-toggles {
-    flex-grow: 1;
+  flex-grow: 1;
-    display: grid;
+  display: grid;
-    grid-template-columns: repeat(6, auto);
+  grid-template-columns: repeat(6, auto);
-    grid-column-gap: 10px;
+  grid-column-gap: 10px;
 }
 
 .profile-tabs {
-    @include search-resize(820px, 5);
+  @include search-resize(820px, 5);
-    @include search-resize(725px, 4);
+  @include search-resize(725px, 4);
-    @include search-resize(600px, 6);
+  @include search-resize(600px, 6);
-    @include search-resize(560px, 5);
+  @include search-resize(560px, 5);
-    @include search-resize(480px, 4);
+  @include search-resize(480px, 4);
-    @include search-resize(410px, 3);
+  @include search-resize(410px, 3);
 }
 
 @include search-resize(560px, 5);

src/sass/tweet/video.scss

@@ -1,66 +1,64 @@
-@import '_variables';
+@import "_variables";
-@import '_mixins';
+@import "_mixins";
 
 video {
-    max-height: 100%;
+  height: 100%;
-    width: 100%;
+  width: 100%;
 }
 
 .gallery-video {
-    display: flex;
+  display: flex;
-    overflow: hidden;
+  overflow: hidden;
 }
 
 .gallery-video.card-container {
-    flex-direction: column;
+  flex-direction: column;
+  width: 100%;
 }
 
 .video-container {
-    max-height: 530px;
+  max-height: 530px;
-    margin: 0;
+  margin: 0;
-    display: flex;
-    align-items: center;
-    justify-content: center;
 
-    img {
+  img {
-        max-height: 100%;
+    max-height: 100%;
-        max-width: 100%;
+    max-width: 100%;
-    }
+  }
 }
 
 .video-overlay {
-    @include play-button;
+  @include play-button;
-    background-color: $shadow;
+  background-color: $shadow;
 
-    p {
+  p {
-        position: relative;
+    position: relative;
-        z-index: 0;
+    z-index: 0;
-        text-align: center;
+    text-align: center;
-        top: calc(50% - 20px);
+    top: calc(50% - 20px);
-        font-size: 20px;
+    font-size: 20px;
-        line-height: 1.3;
+    line-height: 1.3;
-        margin: 0 20px;
+    margin: 0 20px;
-    }
+  }
 
-    div {
+  div {
-        position: relative;
+    position: relative;
-        z-index: 0;
+    z-index: 0;
-        top: calc(50% - 20px);
+    top: calc(50% - 20px);
-        margin: 0 auto;
+    margin: 0 auto;
-        width: 40px;
+    width: 40px;
-        height: 40px;
+    height: 40px;
-    }
+  }
 
-    form {
+  form {
-        width: 100%;
+    width: 100%;
-        height: 100%;
+    height: 100%;
-        align-items: center;
+    align-items: center;
-        justify-content: center;
+    justify-content: center;
-        display: flex;
+    display: flex;
-    }
+  }
 
-    button {
+  button {
-        padding: 5px 8px;
+    padding: 5px 8px;
-        font-size: 16px;
+    font-size: 16px;
-    }
+  }
 }

src/tid.nim

@@ -0,0 +1,62 @@
+import std/[asyncdispatch, base64, httpclient, random, strutils, sequtils, times]
+import nimcrypto
+import experimental/parser/tid
+
+randomize()
+
+const defaultKeyword = "obfiowerehiring";
+const pairsUrl =
+  "https://raw.githubusercontent.com/fa0311/x-client-transaction-id-pair-dict/refs/heads/main/pair.json";
+
+var
+  cachedPairs: seq[TidPair] = @[]
+  lastCached = 0
+  # refresh every hour
+  ttlSec = 60 * 60
+
+proc getPair(): Future[TidPair] {.async.} =
+  if cachedPairs.len == 0 or int(epochTime()) - lastCached > ttlSec:
+    lastCached = int(epochTime())
+
+    let client = newAsyncHttpClient()
+    defer: client.close()
+
+    let resp = await client.get(pairsUrl)
+    if resp.status == $Http200:
+      cachedPairs = parseTidPairs(await resp.body)
+
+  return sample(cachedPairs)
+
+proc encodeSha256(text: string): array[32, byte] =
+  let
+    data = cast[ptr byte](addr text[0])
+    dataLen = uint(len(text))
+    digest = sha256.digest(data, dataLen)
+  return digest.data
+
+proc encodeBase64[T](data: T): string =
+  return encode(data).replace("=", "")
+
+proc decodeBase64(data: string): seq[byte] =
+  return cast[seq[byte]](decode(data))
+
+proc genTid*(path: string): Future[string] {.async.} =
+  let 
+    pair = await getPair()
+
+    timeNow = int(epochTime() - 1682924400)
+    timeNowBytes = @[
+      byte(timeNow and 0xff),
+      byte((timeNow shr 8) and 0xff),
+      byte((timeNow shr 16) and 0xff),
+      byte((timeNow shr 24) and 0xff)
+    ]
+
+    data = "GET!" & path & "!" & $timeNow & defaultKeyword & pair.animationKey
+    hashBytes = encodeSha256(data)
+    keyBytes = decodeBase64(pair.verification)
+    bytesArr = keyBytes & timeNowBytes & hashBytes[0 ..< 16] & @[3'u8]
+    randomNum = byte(rand(256))
+    tid = @[randomNum] & bytesArr.mapIt(it xor randomNum)
+
+  return encodeBase64(tid)

src/tokens.nim

@@ -1,168 +0,0 @@
-# SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, times, sequtils, json, random
-import strutils, tables
-import types, consts
-
-const
-  maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
-  maxLastUse = 1.hours   # if a token is unused for 60 minutes, it expires
-  maxAge = 2.hours + 55.minutes  # tokens expire after 3 hours
-  failDelay = initDuration(minutes=30)
-
-var
-  tokenPool: seq[Token]
-  lastFailed: Time
-  enableLogging = false
-
-let headers = newHttpHeaders({"authorization": auth})
-
-template log(str) =
-  if enableLogging: echo "[tokens] ", str
-
-proc getPoolJson*(): JsonNode =
-  var
-    list = newJObject()
-    totalReqs = 0
-    totalPending = 0
-    reqsPerApi: Table[string, int]
-
-  for token in tokenPool:
-    totalPending.inc(token.pending)
-    list[token.tok] = %*{
-      "apis": newJObject(),
-      "pending": token.pending,
-      "init": $token.init,
-      "lastUse": $token.lastUse
-    }
-
-    for api in token.apis.keys:
-      list[token.tok]["apis"][$api] = %token.apis[api]
-
-      let
-        maxReqs =
-          case api
-          of Api.photoRail: 180
-          #of Api.timeline: 187
-          #of Api.userTweets, Api.userTimeline: 300
-          of Api.userTweets: 300
-          of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
-             Api.userTweetsAndReplies, Api.userMedia,
-             Api.userRestId, Api.userScreenName, Api.tweetDetail,
-             Api.tweetResult, Api.search, Api.favorites,
-             Api.retweeters, Api.favoriters, Api.following, Api.followers: 500
-          #of Api.userSearch: 900
-        reqs = maxReqs - token.apis[api].remaining
-
-      reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
-      totalReqs.inc(reqs)
-
-  return %*{
-    "amount": tokenPool.len,
-    "requests": totalReqs,
-    "pending": totalPending,
-    "apis": reqsPerApi,
-    "tokens": list
-  }
-
-proc rateLimitError*(): ref RateLimitError =
-  newException(RateLimitError, "rate limited")
-
-proc fetchToken(): Future[Token] {.async.} =
-  if getTime() - lastFailed < failDelay:
-    raise rateLimitError()
-
-  let client = newAsyncHttpClient(headers=headers)
-
-  try:
-    let
-      resp = await client.postContent(activate)
-      tokNode = parseJson(resp)["guest_token"]
-      tok = tokNode.getStr($(tokNode.getInt))
-      time = getTime()
-
-    return Token(tok: tok, init: time, lastUse: time)
-  except Exception as e:
-    echo "[tokens] fetching token failed: ", e.msg
-    if "Try again" notin e.msg:
-      echo "[tokens] fetching tokens paused, resuming in 30 minutes"
-      lastFailed = getTime()
-  finally:
-    client.close()
-
-proc expired(token: Token): bool =
-  let time = getTime()
-  token.init < time - maxAge or token.lastUse < time - maxLastUse
-
-proc isLimited(token: Token; api: Api): bool =
-  if token.isNil or token.expired:
-    return true
-
-  if api in token.apis:
-    let limit = token.apis[api]
-    return (limit.remaining <= 10 and limit.reset > epochTime().int)
-  else:
-    return false
-
-proc isReady(token: Token; api: Api): bool =
-  not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
-
-proc release*(token: Token; used=false; invalid=false) =
-  if token.isNil: return
-  if invalid or token.expired:
-    if invalid: log "discarding invalid token"
-    elif token.expired: log "discarding expired token"
-
-    let idx = tokenPool.find(token)
-    if idx > -1: tokenPool.delete(idx)
-  elif used:
-    dec token.pending
-    token.lastUse = getTime()
-
-proc getToken*(api: Api): Future[Token] {.async.} =
-  for i in 0 ..< tokenPool.len:
-    if result.isReady(api): break
-    release(result)
-    result = tokenPool.sample()
-
-  if not result.isReady(api):
-    release(result)
-    result = await fetchToken()
-    log "added new token to pool"
-    tokenPool.add result
-
-  if not result.isNil:
-    inc result.pending
-  else:
-    raise rateLimitError()
-
-proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
-  # avoid undefined behavior in race conditions
-  if api in token.apis:
-    let limit = token.apis[api]
-    if limit.reset >= reset and limit.remaining < remaining:
-      return
-
-  token.apis[api] = RateLimit(remaining: remaining, reset: reset)
-
-proc poolTokens*(amount: int) {.async.} =
-  var futs: seq[Future[Token]]
-  for i in 0 ..< amount:
-    futs.add fetchToken()
-
-  for token in futs:
-    var newToken: Token
-
-    try: newToken = await token
-    except: discard
-
-    if not newToken.isNil:
-      log "added new token to pool"
-      tokenPool.add newToken
-
-proc initTokenPool*(cfg: Config) {.async.} =
-  enableLogging = cfg.enableDebug
-
-  while true:
-    if tokenPool.countIt(not it.isLimited(Api.userTweets)) < cfg.minTokens:
-      await poolTokens(min(4, cfg.minTokens - tokenPool.len))
-    await sleepAsync(2000)

src/types.nim

@@ -12,25 +12,13 @@ type
   TimelineKind* {.pure.} = enum
     tweets, replies, media
 
-  Api* {.pure.} = enum
+  ApiUrl* = object
-    tweetDetail
+    endpoint*: string
-    tweetResult
+    params*: seq[(string, string)]
-    photoRail
+
-    search
+  ApiReq* = object
-    list
+    oauth*: ApiUrl
-    listBySlug
+    cookie*: ApiUrl
-    listMembers
-    listTweets
-    userRestId
-    userScreenName
-    favorites
-    userTweets
-    userTweetsAndReplies
-    userMedia
-    favoriters
-    retweeters
-    following
-    followers
 
   RateLimit* = object
     remaining*: int
@@ -43,14 +31,14 @@ type
     init*: Time
     lastUse*: Time
     pending*: int
-    apis*: Table[Api, RateLimit]
+    apis*: Table[string, RateLimit]
 
   GuestAccount* = ref object
     id*: int64
     oauthToken*: string
     oauthSecret*: string
     pending*: int
-    apis*: Table[Api, RateLimit]
+    apis*: Table[string, RateLimit]
 
   Error* = enum
     null = 0
@@ -308,6 +296,7 @@ type
     enableDebug*: bool
     proxy*: string
     proxyAuth*: string
+    disableTid*: bool
     cookieHeader*: string
     xCsrfToken*: string
 

src/views/renderutils.nim

@@ -26,7 +26,9 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
 template verifiedIcon*(user: User): untyped {.dirty.} =
   if user.verifiedType != VerifiedType.none:
     let lower = ($user.verifiedType).toLowerAscii()
-    icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account")
+    buildHtml(tdiv(class=(&"verified-icon {lower}"))):
+      icon "circle", class="verified-icon-circle", title=(&"Verified {lower} account")
+      icon "ok", class="verified-icon-check", title=(&"Verified {lower} account")
   else:
     text ""
 

src/views/rss.nimf

@@ -2,6 +2,9 @@
 ## SPDX-License-Identifier: AGPL-3.0-only
 #import strutils, xmltree, strformat, options, unicode
 #import ../types, ../utils, ../formatters, ../prefs
+## Snowflake ID cutoff for RSS GUID format transition
+## Corresponds to approximately December 14, 2025 UTC
+#const guidCutoff = 2000000000000000000'i64
 #
 #proc getTitle(tweet: Tweet; retweet: string): string =
 #if tweet.pinned: result = "Pinned: "
@@ -25,7 +28,25 @@
 #end proc
 #
 #proc getDescription(desc: string; cfg: Config): string =
-Twitter feed for: ${desc}. Generated by ${cfg.hostname}
+Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
+#end proc
+#
+#proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
+#result = profile.tweets.content
+#if profile.pinned.isSome and result.len > 0:
+#  let pinnedTweet = profile.pinned.get
+#  var inserted = false
+#  for threadIdx in 0 ..< result.len:
+#    if not inserted:
+#      for tweetIdx in 0 ..< result[threadIdx].len:
+#        if result[threadIdx][tweetIdx].id < pinnedTweet.id:
+#          result[threadIdx].insert(pinnedTweet, tweetIdx)
+#          inserted = true
+#        end if
+#      end for
+#    end if
+#  end for
+#end if
 #end proc
 #
 #proc renderRssTweet(tweet: Tweet; cfg: Config): string =
@@ -38,23 +59,34 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
 <img src="${urlPrefix}${getPicUrl(photo.url)}" style="max-width:250px;" />
 #  end for
 #elif tweet.video.isSome:
-<img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
+<a href="${urlPrefix}${tweet.getLink}">
+<br>Video<br>
+  <img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
+</a>
 #elif tweet.gif.isSome:
 #  let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
 #  let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
 <video poster="${thumb}" autoplay muted loop style="max-width:250px;">
   <source src="${url}" type="video/mp4"></video>
-##elif tweet.card.isSome:
+#elif tweet.card.isSome:
-##  let card = tweet.card.get()
+#  let card = tweet.card.get()
-##  if card.image.len > 0:
+#  if card.image.len > 0:
-##<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
+<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
-##  end if
+#  end if
 #end if
 #if tweet.quote.isSome and get(tweet.quote).available:
-#  let quoteLink = getLink(get(tweet.quote))
+#  let quoteTweet = get(tweet.quote)
+#  let quoteLink = urlPrefix & getLink(quoteTweet)
 <hr/>
-<p>Quoting: <a href="${urlPrefix}${quoteLink}">${cfg.hostname}${quoteLink}</a></p>
+<blockquote>
-${renderRssTweet(get(tweet.quote), cfg)}
+<b>${quoteTweet.user.fullname} (@${quoteTweet.user.username})</b>
+<p>
+${renderRssTweet(quoteTweet, cfg)}
+</p>
+<footer>
+— <cite><a href="${quoteLink}">${quoteLink}</a>
+</footer>
+</blockquote>
 #end if
 #end proc
 #
@@ -72,12 +104,17 @@ ${renderRssTweet(get(tweet.quote), cfg)}
 #    if link in links: continue
 #    end if
 #    links.add link
+#    let useGlobalGuid = parseBiggestInt(tweet.id) >= guidCutoff
       <item>
         <title>${getTitle(tweet, retweet)}</title>
         <dc:creator>@${tweet.user.username}</dc:creator>
         <description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
         <pubDate>${getRfc822Time(tweet)}</pubDate>
+#if useGlobalGuid:
+        <guid isPermaLink="false">${tweet.id}</guid>
+#else:
         <guid>${urlPrefix & link}</guid>
+#end if
         <link>${urlPrefix & link}</link>
       </item>
 #  end for
@@ -108,8 +145,9 @@ ${renderRssTweet(get(tweet.quote), cfg)}
       <width>128</width>
       <height>128</height>
     </image>
-#if profile.tweets.content.len > 0:
+#let tweetsList = getTweetsWithPinned(profile)
-${renderRssTweets(profile.tweets.content, cfg, userId=profile.user.id)}
+#if tweetsList.len > 0:
+${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
 #end if
   </channel>
 </rss>

src/views/status.nim

@@ -28,14 +28,19 @@ proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode =
     if thread.hasMore:
       renderMoreReplies(thread)
 
-proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode =
+proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string; tweet: Tweet = nil): VNode =
   buildHtml(tdiv(class="replies", id="r")):
+    var hasReplies = false
+    var replyCount = 0
     for thread in replies.content:
       if thread.content.len == 0: continue
+      hasReplies = true
+      replyCount += thread.content.len
       renderReplyThread(thread, prefs, path)
 
-    if replies.bottom.len > 0:
+    if hasReplies and replies.bottom.len > 0:
-      renderMore(Query(), replies.bottom, focus="#r")
+      if tweet == nil or not replies.beginning or replyCount < tweet.stats.replies:
+        renderMore(Query(), replies.bottom, focus="#r")
 
 proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode =
   let hasAfter = conv.after.content.len > 0
@@ -70,6 +75,6 @@ proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode
       if not conv.replies.beginning:
         renderNewer(Query(), getLink(conv.tweet), focus="#r")
       if conv.replies.content.len > 0 or conv.replies.bottom.len > 0:
-        renderReplies(conv.replies, prefs, path)
+        renderReplies(conv.replies, prefs, path, conv.tweet)
 
     renderToTop(focus="#m")