From 0bc6b3325123af0e6be8046e792bb88f09f7a7ef Mon Sep 17 00:00:00 2001 From: Cynthia Foxwell Date: Sat, 6 Dec 2025 17:12:00 -0700 Subject: [PATCH] bunch of upstream changes --- nitter.example.conf | 7 +- src/api.nim | 193 ++++++++++---------- src/apiutils.nim | 95 +++------- src/auth.nim | 6 +- src/config.nim | 1 + src/consts.nim | 168 +++++++----------- src/experimental/parser/graphql.nim | 64 +++++-- src/experimental/parser/tid.nim | 8 + src/experimental/types/graphuser.nim | 41 ++++- src/experimental/types/tid.nim | 4 + src/nitter.nim | 4 +- src/parser.nim | 46 +++-- src/redis_cache.nim | 15 +- src/routes/activityspoof.nim | 14 +- src/routes/debug.nim | 17 -- src/routes/status.nim | 2 +- src/routes/timeline.nim | 3 +- src/routes/twitter_api.nim | 88 ++++----- src/sass/general.scss | 1 + src/sass/index.scss | 256 +++++++++++++++------------ src/sass/navbar.scss | 120 ++++++------- src/sass/search.scss | 171 +++++++++--------- src/sass/tweet/video.scss | 90 +++++----- src/tid.nim | 62 +++++++ src/tokens.nim | 168 ------------------ src/types.nim | 31 ++-- src/views/renderutils.nim | 4 +- src/views/rss.nimf | 62 +++++-- src/views/status.nim | 13 +- 29 files changed, 841 insertions(+), 913 deletions(-) create mode 100644 src/experimental/parser/tid.nim create mode 100644 src/experimental/types/tid.nim delete mode 100644 src/routes/debug.nim create mode 100644 src/tid.nim delete mode 100644 src/tokens.nim diff --git a/nitter.example.conf b/nitter.example.conf index f0b4214..100d875 100644 --- a/nitter.example.conf +++ b/nitter.example.conf @@ -26,12 +26,7 @@ enableRSS = true # set this to false to disable RSS feeds enableDebug = false # enable request logs and debug endpoints (/.accounts) proxy = "" # http/https url, SOCKS proxies are not supported proxyAuth = "" -tokenCount = 10 -# minimum amount of usable tokens. tokens are used to authorize API requests, -# but they expire after ~1 hour, and have a limit of 500 requests per endpoint. -# the limits reset every 15 minutes, and the pool is filled up so there's -# always at least `tokenCount` usable tokens. only increase this if you receive -# major bursts all the time and don't have a rate limiting setup via e.g. nginx +disableTid = false # enable this if cookie-based auth is failing # Change default preferences here, see src/prefs_impl.nim for a complete list [Preferences] diff --git a/src/api.nim b/src/api.nim index 52ef057..5e837c4 100644 --- a/src/api.nim +++ b/src/api.nim @@ -1,78 +1,114 @@ # SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, httpclient, uri, strutils, sequtils, sugar +import asyncdispatch, httpclient, strutils, sequtils, sugar import packedjson import types, query, formatters, consts, apiutils, parser import experimental/parser as newParser +# Helper to generate params object for GraphQL requests +proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] = + result.add ("variables", variables) + result.add ("features", gqlFeatures) + if fieldToggles.len > 0: + result.add ("fieldToggles", fieldToggles) + +proc apiUrl*(endpoint, variables: string; fieldToggles = ""): ApiUrl = + return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles)) + +proc apiReq*(endpoint, variables: string; fieldToggles = ""): ApiReq = + let url = apiUrl(endpoint, variables, fieldToggles) + return ApiReq(cookie: url, oauth: url) + +proc mediaUrl*(id: string; cursor: string): ApiReq = + result = ApiReq( + cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor], """{"withArticlePlainText":false}"""), + oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor]) + ) + +proc userTweetsUrl*(id: string; cursor: string): ApiReq = + result = ApiReq( + cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles), + oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor]) + ) + +proc userTweetsAndRepliesUrl*(id: string; cursor: string): ApiReq = + let cookieVars = userTweetsAndRepliesVars % [id, cursor] + result = ApiReq( + cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles), + oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor]) + ) + +proc tweetDetailUrl*(id: string; cursor: string): ApiReq = + result = ApiReq( + cookie: apiUrl(graphTweetDetail, tweetDetailVars % [id, cursor], tweetDetailFieldToggles), + oauth: apiUrl(graphTweet, tweetVars % [id, cursor]) + ) + +proc userUrl*(username: string): ApiReq = + let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username + result = ApiReq( + cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles), + oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username) + ) + proc getGraphUser*(username: string): Future[User] {.async.} = if username.len == 0: return - let - headers = newHttpHeaders() + let headers = newHttpHeaders() headers.add("Referer", """https://x.com/$1""" % username) - let - variables = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username - fieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}""" - params = {"variables": variables, "features": userFeatures, "fieldToggles": fieldToggles} - js = await fetch(graphUser ? params, Api.userScreenName, headers) + let js = await fetchRaw(userUrl(username), headers) result = parseGraphUser(js) proc getGraphUserById*(id: string): Future[User] {.async.} = if id.len == 0 or id.any(c => not c.isDigit): return - let - headers = newHttpHeaders() + let headers = newHttpHeaders() headers.add("Referer", """https://x.com/i/user/$1""" % id) let - variables = """{"userId":"$1"}""" % id - params = {"variables": variables, "features": userFeatures} - js = await fetch(graphUserById ? params, Api.userRestId, headers) + url = apiReq(graphUserById, """{"userId":"$1"}""" % id) + js = await fetchRaw(url, headers) result = parseGraphUser(js) proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} = if id.len == 0: return - let endpoint = case kind - of TimelineKind.tweets: "" - of TimelineKind.replies: "/with_replies" - of TimelineKind.media: "/media" let + endpoint = case kind + of TimelineKind.tweets: "" + of TimelineKind.replies: "/with_replies" + of TimelineKind.media: "/media" headers = newHttpHeaders() headers.add("Referer", """https://x.com/$1$2""" % [id, endpoint]) let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = if kind == TimelineKind.media: userMediaVariables % [id, cursor] else: userTweetsVariables % [id, cursor] - fieldToggles = """{"withArticlePlainText":true}""" - params = {"variables": variables, "features": gqlFeatures, "fieldToggles": fieldToggles} - (url, apiId) = case kind - of TimelineKind.tweets: (graphUserTweets, Api.userTweets) - of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies) - of TimelineKind.media: (graphUserMedia, Api.userMedia) - js = await fetch(url ? params, apiId, headers) + url = case kind + of TimelineKind.tweets: userTweetsUrl(id, cursor) + of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor) + of TimelineKind.media: mediaUrl(id, cursor) + js = await fetch(url, headers) result = parseGraphTimeline(js, if kind == TimelineKind.media: "" else: "user", after) proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} = if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = listTweetsVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - js = await fetch(graphListTweets ? params, Api.listTweets) + url = apiReq(graphListTweets, restIdVars % [id, cursor]) + js = await fetch(url) result = parseGraphTimeline(js, "list", after).tweets proc getGraphListBySlug*(name, list: string): Future[List] {.async.} = let variables = %*{"screenName": name, "listSlug": list} - params = {"variables": $variables, "features": gqlFeatures} - result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug)) + url = apiReq(graphListBySlug, $variables) + js = await fetch(url) + result = parseGraphList(js) proc getGraphList*(id: string): Future[List] {.async.} = - let - variables = """{"listId":"$1"}""" % id - params = {"variables": variables, "features": gqlFeatures} - result = parseGraphList(await fetch(graphListById ? params, Api.list)) + let + url = apiReq(graphListById, """{"listId": "$1"}""" % id) + js = await fetch(url) + result = parseGraphList(js) proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} = if list.id.len == 0: return @@ -86,85 +122,45 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} } if after.len > 0: variables["cursor"] = % after - let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures} - result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after) - -proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} = - if id.len == 0: return - var - variables = %*{ - "userId": id, - "includePromotedContent":false, - "withClientEventToken":false, - "withBirdwatchNotes":true, - "withVoice":true, - "withV2Timeline":false - } - if after.len > 0: - variables["cursor"] = % after - let - url = consts.favorites ? {"variables": $variables, "features": gqlFeatures} - result = parseGraphTimeline(await fetch(url, Api.favorites), after) + let + url = apiReq(graphListMembers, $variables) + js = await fetchRaw(url) + result = parseGraphListMembers(js, after) proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} = if id.len == 0: return - let - headers = newHttpHeaders() + + let headers = newHttpHeaders() headers.add("Referer", """https://x.com/i/status/$1""" % id) let - variables = """{"rest_id":"$1"}""" % id - params = {"variables": variables, "features": gqlFeatures} - js = await fetch(graphTweetResult ? params, Api.tweetResult, headers) + url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id) + js = await fetch(url, headers) result = parseGraphTweetResult(js) proc getGraphTweet*(id: string; after=""): Future[Conversation] {.async.} = if id.len == 0: return - let - headers = newHttpHeaders() + + let headers = newHttpHeaders() headers.add("Referer", """https://x.com/i/status/$1""" % id) let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = tweetVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures, "fieldToggles": tweetFieldToggles} - js = await fetch(graphTweet ? params, Api.tweetDetail, headers) + js = await fetch(tweetDetailUrl(id, cursor), headers) result = parseGraphConversation(js, id) -proc getGraphFavoriters*(id: string; after=""): Future[UsersTimeline] {.async.} = - if id.len == 0: return - let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = reactorsVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - js = await fetch(graphFavoriters ? params, Api.favoriters) - result = parseGraphFavoritersTimeline(js, id) - -proc getGraphRetweeters*(id: string; after=""): Future[UsersTimeline] {.async.} = - if id.len == 0: return - let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = reactorsVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - js = await fetch(graphRetweeters ? params, Api.retweeters) - result = parseGraphRetweetersTimeline(js, id) - proc getGraphFollowing*(id: string; after=""): Future[UsersTimeline] {.async.} = if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = followVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - js = await fetch(graphFollowing ? params, Api.following) + js = await fetch(apiReq(graphFollowing, followVars % [id, cursor])) result = parseGraphFollowTimeline(js, id) proc getGraphFollowers*(id: string; after=""): Future[UsersTimeline] {.async.} = if id.len == 0: return let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = followVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - js = await fetch(graphFollowers ? params, Api.followers) + js = await fetch(apiReq(graphFollowers, followVars % [id, cursor])) result = parseGraphFollowTimeline(js, id) proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} = @@ -191,8 +187,10 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = } if after.len > 0: variables["cursor"] = % after - let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} - result = parseGraphSearch[Tweets](await fetch(url, Api.search), after) + let + url = apiReq(graphSearchTimeline, $variables) + js = await fetch(url) + result = parseGraphSearch[Tweets](js, after) result.query = query proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} = @@ -211,17 +209,16 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} variables["cursor"] = % after result.beginning = false - let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} - result = parseGraphSearch[User](await fetch(url, Api.search), after) + let + url = apiReq(graphSearchTimeline, $variables) + js = await fetch(url) + result = parseGraphSearch[User](js, after) result.query = query -proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} = - if name.len == 0: return - let - ps = genParams({"screen_name": name, "trim_user": "true"}, - count="18", ext=false) - url = photoRail ? ps - result = parsePhotoRail(await fetch(url, Api.photoRail)) +proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} = + if id.len == 0: return + let js = await fetch(mediaUrl(id, "")) + result = parseGraphPhotoRail(js) proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} = let client = newAsyncHttpClient(maxRedirects=0) diff --git a/src/apiutils.nim b/src/apiutils.nim index e606ff5..535f3bd 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only import httpclient, asyncdispatch, options, strutils, uri, times, tables, math import jsony, packedjson, zippy -import types, tokens, consts, parserutils, http_pool +import types, consts, parserutils, http_pool, tid import experimental/types/common import config @@ -9,42 +9,34 @@ const rlRemaining = "x-rate-limit-remaining" rlReset = "x-rate-limit-reset" -var pool: HttpPool +var + pool: HttpPool + disableTid: bool -proc genParams*(pars: openArray[(string, string)] = @[]; cursor=""; - count="20"; ext=true): seq[(string, string)] = - result = timelineParams - for p in pars: - result &= p - if ext: - result &= ("include_ext_alt_text", "1") - result &= ("include_ext_media_stats", "1") - result &= ("include_ext_media_availability", "1") - if count.len > 0: - result &= ("count", count) - if cursor.len > 0: - # The raw cursor often has plus signs, which sometimes get turned into spaces, - # so we need to turn them back into a plus - if " " in cursor: - result &= ("cursor", cursor.replace(" ", "+")) - else: - result &= ("cursor", cursor) +proc setDisableTid*(disable: bool) = + disableTid = disable + +proc toUrl(req: ApiReq): Uri = + let c = req.cookie + parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params + +proc rateLimitError*(): ref RateLimitError = + newException(RateLimitError, "rate limited") -#proc genHeaders*(token: Token = nil): HttpHeaders = proc genHeaders*(): HttpHeaders = let t = getTime() ffVersion = floor(124 + (t.toUnix() - 1710892800) / 2419200) result = newHttpHeaders({ "Connection": "keep-alive", - "Authorization": auth, + "Authorization": bearerToken, "Content-Type": "application/json", "Accept-Encoding": "gzip", "Accept-Language": "en-US,en;q=0.5", "Accept": "*/*", "DNT": "1", "Host": "x.com", - "Origin": "https://x.com/", + "Origin": "https://x.com", "Sec-Fetch-Dest": "empty", "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-origin", @@ -56,30 +48,20 @@ proc genHeaders*(): HttpHeaders = "x-twitter-client-language": "en" }, true) -#template updateToken() = -# if resp.headers.hasKey(rlRemaining): -# let -# remaining = parseInt(resp.headers[rlRemaining]) -# reset = parseInt(resp.headers[rlReset]) -# token.setRateLimit(api, remaining, reset) - template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = once: pool = HttpPool() - #var token = await getToken(api) - #if token.tok.len == 0: - # raise rateLimitError() - if len(cfg.cookieHeader) != 0: additional_headers.add("Cookie", cfg.cookieHeader) + if not disableTid: + additional_headers.add("x-client-transaction-id", await genTid(url.path)) if len(cfg.xCsrfToken) != 0: additional_headers.add("x-csrf-token", cfg.xCsrfToken) try: var resp: AsyncResponse - #var headers = genHeaders(token) var headers = genHeaders() for key, value in additional_headers.pairs(): headers.add(key, value) @@ -102,7 +84,6 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = let remaining = parseInt(resp.headers[rlRemaining]) reset = parseInt(resp.headers[rlReset]) - #token.setRateLimit(api, remaining, reset) if result.len > 0: if resp.headers.getOrDefault("content-encoding") == "gzip": @@ -111,39 +92,27 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = if result.startsWith("{\"errors"): let errors = result.fromJson(Errors) if errors in {expiredToken, badToken, authorizationError}: - echo "fetch error: ", errors - #release(token, invalid=true) raise rateLimitError() elif errors in {rateLimited}: - # rate limit hit, resets after 24 hours - #setLimited(account, api) raise rateLimitError() elif result.startsWith("429 Too Many Requests"): - #echo "[accounts] 429 error, API: ", api, ", token: ", token[] - #account.apis[api].remaining = 0 - # rate limit hit, resets after the 15 minute window raise rateLimitError() fetchBody - #release(token, used=true) - if resp.status == $Http400: - let errText = "body: '" & result & "' url: " & $url - raise newException(InternalError, errText) + echo "ERROR 400, ", url.path, ": ", result + raise newException(InternalError, $url) except InternalError as e: raise e except BadClientError as e: - #release(token, used=true) raise e except OSError as e: raise e except ProtocolError as e: raise e except Exception as e: - echo "error: ", e.name, ", msg: ", e.msg, ", url: ", url - #if "length" notin e.msg and "descriptor" notin e.msg: - #release(token, invalid=true) + echo "error: ", e.name, ", msg: ", e.msg, ", url: ", url.path raise rateLimitError() template retry(bod) = @@ -152,36 +121,26 @@ template retry(bod) = except ProtocolError: bod -proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = +proc fetch*(req: ApiReq; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} = retry: var body: string + let url = req.toUrl() fetchImpl(body, additional_headers): if body.startsWith('{') or body.startsWith('['): result = parseJson(body) else: - echo resp.status, ": ", body, " --- url: ", url + echo resp.status, " - non-json for: ", url, ", body: ", result result = newJNull() - #updateToken() - let error = result.getError if error in {expiredToken, badToken}: - echo "fetch error: ", result.getError - #release(token, invalid=true) + echo "Fetch error, API: ", url.path, ", error: ", result.getError raise rateLimitError() -proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} = +proc fetchRaw*(req: ApiReq; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} = retry: + let url = req.toUrl() fetchImpl(result, additional_headers): if not (result.startsWith('{') or result.startsWith('[')): - echo resp.status, ": ", result, " --- url: ", url + echo resp.status, " - non-json for: ", url, ", body: ", result result.setLen(0) - - #updateToken() - - if result.startsWith("{\"errors"): - let errors = result.fromJson(Errors) - if errors in {expiredToken, badToken}: - echo "fetch error: ", errors - #release(token, invalid=true) - raise rateLimitError() diff --git a/src/auth.nim b/src/auth.nim index de1b1d8..25648cd 100644 --- a/src/auth.nim +++ b/src/auth.nim @@ -126,7 +126,7 @@ proc getAccountPoolDebug*(): JsonNode = proc rateLimitError*(): ref RateLimitError = newException(RateLimitError, "rate limited") -proc isLimited(account: GuestAccount; api: Api): bool = +proc isLimited(account: GuestAccount; req: ApiReq): bool = if account.isNil: return true @@ -157,9 +157,9 @@ proc release*(account: GuestAccount) = if account.isNil: return dec account.pending -proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} = +proc getGuestAccount*(req: ApiReq): Future[GuestAccount] {.async.} = for i in 0 ..< accountPool.len: - if result.isReady(api): break + if result.isReady(req): break result = accountPool.sample() if not result.isNil and result.isReady(api): diff --git a/src/config.nim b/src/config.nim index fe4aba5..7fdf83d 100644 --- a/src/config.nim +++ b/src/config.nim @@ -42,6 +42,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) = enableDebug: cfg.get("Config", "enableDebug", false), proxy: cfg.get("Config", "proxy", ""), proxyAuth: cfg.get("Config", "proxyAuth", ""), + disableTid: cfg.get("Config", "disableTid", false), cookieHeader: cfg.get("Config", "cookieHeader", ""), xCsrfToken: cfg.get("Config", "xCsrfToken", "") ) diff --git a/src/consts.nim b/src/consts.nim index 7326540..e7127fd 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -1,54 +1,30 @@ # SPDX-License-Identifier: AGPL-3.0-only -import uri, sequtils, strutils +import strutils const - auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" - consumerKey* = "3nVuSoBZnx6U4vzUxf5w" consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" + bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" - api = parseUri("https://x.com/i/api") - activate* = $(api / "1.1/guest/activate.json") - - photoRail* = api / "1.1/statuses/media_timeline.json" - - timelineApi = api / "2/timeline" - - graphql = api / "graphql" - graphUser* = graphql / "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName" - graphUserById* = graphql / "Bbaot8ySMtJD7K2t01gW7A/UserByRestId" - graphUserTweets* = graphql / "lZRf8IC-GTuGxDwcsHW8aw/UserTweets" - graphUserTweetsAndReplies* = graphql / "gXCeOBFsTOuimuCl1qXimg/UserTweetsAndReplies" - graphUserMedia* = graphql / "1D04dx9H2pseMQAbMjXTvQ/UserMedia" - graphTweet* = graphql / "6QzqakNMdh_YzBAR9SYPkQ/TweetDetail" - graphTweetResult* = graphql / "kLXoXTloWpv9d2FSXRg-Tg/TweetResultByRestId" - graphTweetHistory* = graphql / "WT7HhrzWulh4yudKJaR10Q/TweetEditHistory" - graphSearchTimeline* = graphql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline" - graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId" - graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug" - graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers" - graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" - graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters" - graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters" - graphFollowers* = graphql / "SCu9fVIlCUm-BM8-tL5pkQ/Followers" - graphFollowing* = graphql / "S5xUN9s2v4xk50KWGGvyvQ/Following" - favorites* = graphql / "eSSNbhECHHWWALkkQq-YTA/Likes" - - timelineParams* = { - "include_can_media_tag": "1", - "include_cards": "1", - "include_entities": "1", - "include_profile_interstitial_type": "0", - "include_quote_count": "0", - "include_reply_count": "0", - "include_user_entities": "0", - "include_ext_reply_count": "0", - "include_ext_media_color": "0", - "cards_platform": "Web-13", - "tweet_mode": "extended", - "send_error_codes": "1", - "simple_quoted_tweet": "1" - }.toSeq + graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName" + graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery" + graphUserById* = "Bbaot8ySMtJD7K2t01gW7A/UserByRestId" + graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2" + graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2" + graphUserTweets* = "lZRf8IC-GTuGxDwcsHW8aw/UserTweets" + graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies" + graphUserMedia* = "1D04dx9H2pseMQAbMjXTvQ/UserMedia" + graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2" + graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline" + graphTweetDetail* = "6QzqakNMdh_YzBAR9SYPkQ/TweetDetail" + graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery" + graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline" + graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId" + graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug" + graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers" + graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline" + graphFollowers* = "Efm7xwLreAw77q2Fq7rX-Q/Followers" + graphFollowing* = "e0UtTAwQqgLKBllQxMgVxQ/Following" gqlFeatures* = """{ "rweb_video_screen_enabled": false, @@ -84,85 +60,77 @@ const "responsive_web_grok_image_annotation_enabled": true, "responsive_web_grok_imagine_annotation_enabled": true, "responsive_web_grok_community_note_auto_translation_is_enabled": false, - "responsive_web_enhance_cards_enabled": false + "responsive_web_enhance_cards_enabled": false, + "payments_enabled": false, + "responsive_web_twitter_article_notes_tab_enabled": false, + "hidden_profile_subscriptions_enabled": false, + "subscriptions_verification_info_verified_since_enabled": false, + "subscriptions_verification_info_is_identity_verified_enabled": false, + "highlights_tweets_tab_ui_enabled": false, + "subscriptions_feature_can_gift_premium": false }""".replace(" ", "").replace("\n", "") - userFeatures* = """{ - "hidden_profile_subscriptions_enabled": true, - "profile_label_improvements_pcf_label_in_post_enabled": true, - "responsive_web_profile_redirect_enabled": false, - "rweb_tipjar_consumption_enabled": true, - "verified_phone_label_enabled": false, - "subscriptions_verification_info_is_identity_verified_enabled": true, - "subscriptions_verification_info_verified_since_enabled": true, - "highlights_tweets_tab_ui_enabled": true, - "responsive_web_twitter_article_notes_tab_enabled": true, - "subscriptions_feature_can_gift_premium": true, - "creator_subscriptions_tweet_preview_api_enabled": true, - "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, - "responsive_web_graphql_timeline_navigation_enabled": true + tweetVars* = """{ + "postId": "$1", + $2 + "includeHasBirdwatchNotes": false, + "includePromotedContent": false, + "withBirdwatchNotes": false, + "withVoice": false, + "withV2Timeline": true }""".replace(" ", "").replace("\n", "") - tweetVariables* = """{ + tweetDetailVars* = """{ "focalTweetId": "$1", $2 + "referrer": "profile", "with_rux_injections": false, "rankingMode": "Relevance", - "includePromotedContent": false, + "includePromotedContent": true, "withCommunity": true, - "withQuickPromoteEligibilityTweetFields": false, + "withQuickPromoteEligibilityTweetFields": true, "withBirdwatchNotes": true, "withVoice": true }""".replace(" ", "").replace("\n", "") - tweetFieldToggles* = """{ - "withArticleRichContentState": true, - "withArticlePlainText": false, - "withGrokAnalyze": false, - "withDisallowedReplyControls": false + restIdVars* = """{ + "rest_id": "$1", $2 + "count": 20 +}""" + + userMediaVars* = """{ + "userId": "$1", $2 + "count": 20, + "includePromotedContent": false, + "withClientEventToken": false, + "withBirdwatchNotes": false, + "withVoice": true }""".replace(" ", "").replace("\n", "") - userTweetsVariables* = """{ - "userId": "$1", - $2 + userTweetsVars* = """{ + "userId": "$1", $2 + "count": 20, + "includePromotedContent": false, + "withQuickPromoteEligibilityTweetFields": true, + "withVoice": true +}""".replace(" ", "").replace("\n", "") + + userTweetsAndRepliesVars* = """{ + "userId": "$1", $2 "count": 20, "includePromotedContent": false, "withCommunity": true, "withVoice": true }""".replace(" ", "").replace("\n", "") - listTweetsVariables* = """{ - "rest_id": "$1", - $2 - "count": 20 -}""".replace(" ", "").replace("\n", "") - - reactorsVariables* = """{ - "tweetId": "$1", + followVars* = """{ + "userId": "$1", $2 "count": 20, "includePromotedContent": false }""".replace(" ", "").replace("\n", "") - followVariables* = """{ - "userId": "$1", - $2 - "count": 20, - "includePromotedContent": false, - "withGrokTranslatedBio": false -}""".replace(" ", "").replace("\n", "") - userMediaVariables* = """{ - "userId": "$1", - $2 - "count": 20, - "includePromotedContent": false, - "withClientEventToken": false, - "withBirdwatchNotes": true, - "withVoice": true -}""".replace(" ", "").replace("\n", "") - - tweetHistoryVariables* = """{ - "tweetId": "$1", - "withQuickPromoteEligibilityTweetFields": false -}""".replace(" ", "").replace("\n", "") + userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}""" + userTweetsFieldToggles* = """{"withArticlePlainText":false}""" + tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}""" diff --git a/src/experimental/parser/graphql.nim b/src/experimental/parser/graphql.nim index 7d84833..65ebc3d 100644 --- a/src/experimental/parser/graphql.nim +++ b/src/experimental/parser/graphql.nim @@ -1,21 +1,53 @@ -import options +import options, strutils import jsony -import user, ../types/[graphuser, graphlistmembers] +import user, utils, ../types/[graphuser, graphlistmembers] from ../../types import User, VerifiedType, Result, Query, QueryKind -#proc parseGraphUser*(json: string): User = -# if json.len == 0 or json[0] != '{': -# return -# -# let raw = json.fromJson(GraphUser) -# -# if raw.data.userResult.result.unavailableReason.get("") == "Suspended": -# return User(suspended: true) -# -# result = raw.data.userResult.result.legacy -# result.id = raw.data.userResult.result.restId -# if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified: -# result.verifiedType = blue +proc parseUserResult*(userResult: UserResult): User = + result = userResult.legacy + + if result.verifiedType == none and userResult.isBlueVerified: + result.verifiedType = blue + + if result.username.len == 0 and userResult.core.screenName.len > 0: + result.id = userResult.restId + result.username = userResult.core.screenName + result.fullname = userResult.core.name + result.userPic = userResult.avatar.imageUrl.replace("_normal", "") + + if userResult.privacy.isSome: + result.protected = userResult.privacy.get.protected + + if userResult.location.isSome: + result.location = userResult.location.get.location + + if userResult.core.createdAt.len > 0: + result.joinDate = parseTwitterDate(userResult.core.createdAt) + + if userResult.verification.isSome: + let v = userResult.verification.get + if v.verifiedType != VerifiedType.none: + result.verifiedType = v.verifiedType + + if userResult.profileBio.isSome and result.bio.len == 0: + result.bio = userResult.profileBio.get.description + +proc parseGraphUser*(json: string): User = + if json.len == 0 or json[0] != '{': + return + + let + raw = json.fromJson(GraphUser) + userResult = + if raw.data.userResult.isSome: raw.data.userResult.get.result + elif raw.data.user.isSome: raw.data.user.get.result + else: UserResult() + + if userResult.unavailableReason.get("") == "Suspended" or + userResult.reason.get("") == "Suspended": + return User(suspended: true) + + result = parseUserResult(userResult) proc parseGraphListMembers*(json, cursor: string): Result[User] = result = Result[User]( @@ -31,7 +63,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] = of TimelineTimelineItem: let userResult = entry.content.itemContent.userResults.result if userResult.restId.len > 0: - result.content.add userResult.legacy + result.content.add parseUserResult(userResult) of TimelineTimelineCursor: if entry.content.cursorType == "Bottom": result.bottom = entry.content.value diff --git a/src/experimental/parser/tid.nim b/src/experimental/parser/tid.nim new file mode 100644 index 0000000..28fccea --- /dev/null +++ b/src/experimental/parser/tid.nim @@ -0,0 +1,8 @@ +import jsony +import ../types/tid +export TidPair + +proc parseTidPairs*(raw: string): seq[TidPair] = + result = raw.fromJson(seq[TidPair]) + if result.len == 0: + raise newException(ValueError, "Parsing pairs failed: " & raw) diff --git a/src/experimental/types/graphuser.nim b/src/experimental/types/graphuser.nim index 08100f9..62c6612 100644 --- a/src/experimental/types/graphuser.nim +++ b/src/experimental/types/graphuser.nim @@ -1,15 +1,48 @@ -import options -from ../../types import User +import options, strutils +from ../../types import User, VerifiedType type GraphUser* = object - data*: tuple[userResult: UserData] + data*: tuple[userResult: Option[UserData], user: Option[UserData]] UserData* = object result*: UserResult - UserResult = object + UserCore* = object + name*: string + screenName*: string + createdAt*: string + + UserBio* = object + description*: string + + UserAvatar* = object + imageUrl*: string + + Verification* = object + verifiedType*: VerifiedType + + Location* = object + location*: string + + Privacy* = object + protected*: bool + + UserResult* = object legacy*: User restId*: string isBlueVerified*: bool + core*: UserCore + avatar*: UserAvatar unavailableReason*: Option[string] + reason*: Option[string] + privacy*: Option[Privacy] + profileBio*: Option[UserBio] + verification*: Option[Verification] + location*: Option[Location] + +proc enumHook*(s: string; v: var VerifiedType) = + v = try: + parseEnum[VerifiedType](s) + except: + VerifiedType.none diff --git a/src/experimental/types/tid.nim b/src/experimental/types/tid.nim new file mode 100644 index 0000000..ad036d9 --- /dev/null +++ b/src/experimental/types/tid.nim @@ -0,0 +1,4 @@ +type + TidPair* = object + animationKey*: string + verification*: string diff --git a/src/nitter.nim b/src/nitter.nim index a813490..0db3980 100644 --- a/src/nitter.nim +++ b/src/nitter.nim @@ -6,7 +6,7 @@ from htmlgen import a import jester -import types, config, prefs, formatters, redis_cache, http_pool +import types, config, prefs, formatters, redis_cache, http_pool, apiutils import views/[general, about] import routes/[ preferences, timeline, status, media, search, list, rss, #debug, @@ -34,6 +34,7 @@ setHmacKey(cfg.hmacKey) setProxyEncoding(cfg.base64Media) setMaxHttpConns(cfg.httpMaxConns) setHttpProxy(cfg.proxy, cfg.proxyAuth) +setDisableTid(cfg.disableTid) initAboutPage(cfg.staticDir) waitFor initRedisPool(cfg) @@ -104,7 +105,6 @@ routes: extend preferences, "" extend resolver, "" extend embed, "" - #extend debug, "" extend activityspoof, "" extend api, "" extend unsupported, "" diff --git a/src/parser.nim b/src/parser.nim index f10fc7e..44e0a37 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -45,6 +45,7 @@ proc parseGraphUser*(js: JsonNode): User = result.fullname = user{"core", "name"}.getStr result.joinDate = parseTwitterDate(user{"core", "created_at"}.getStr) result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "") + result.protected = user{"privacy", "protected"}.getBool let label = user{"affiliates_highlighted_label", "label"} if not label.isNull: @@ -449,23 +450,6 @@ proc parseTimeline*(js: JsonNode; after=""): Profile = else: result.tweets.top = cursor{"value"}.getStr -proc parsePhotoRail*(js: JsonNode): PhotoRail = - with error, js{"error"}: - if error.getStr == "Not authorized.": - return - - for tweet in js: - let - t = parseTweet(tweet, js{"tweet_card"}) - url = if t.photos.len > 0: t.photos[0].url - elif t.video.isSome: get(t.video).thumb - elif t.gif.isSome: get(t.gif).thumb - elif t.card.isSome: get(t.card).image - else: "" - - if url.len == 0: continue - result.add GalleryPhoto(url: url, tweetId: $t.id) - proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet = if js.kind == JNull: return Tweet() @@ -531,6 +515,34 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet = else: result.stats.views = -1 +proc parseGraphPhotoRail*(js: JsonNode): PhotoRail = + result = @[] + + let instructions = + ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"} + + for i in instructions: + if i{"__typename"}.getStr == "TimelineAddEntries": + for e in i{"entries"}: + let entryId = e{"entryId"}.getStr + if entryId.startsWith("tweet"): + with tweetResult, e{"content", "content", "tweetResult", "result"}: + let t = parseGraphTweet(tweetResult, false) + if not t.available: + t.id = $parseBiggestInt(entryId.getId()) + + let url = + if t.photos.len > 0: t.photos[0].url + elif t.video.isSome: get(t.video).thumb + elif t.gif.isSome: get(t.gif).thumb + elif t.card.isSome: get(t.card).image + else: "" + + result.add GalleryPhoto(url: url, tweetId: t.id) + + if result.len == 16: + break + proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] = for t in js{"content", "items"}: let entryId = t{"entryId"}.getStr diff --git a/src/redis_cache.nim b/src/redis_cache.nim index 16f3c25..13b99e3 100644 --- a/src/redis_cache.nim +++ b/src/redis_cache.nim @@ -87,7 +87,7 @@ proc cache*(data: List) {.async.} = await setEx(data.listKey, listCacheTime, compress(toFlatty(data))) proc cache*(data: PhotoRail; name: string) {.async.} = - await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data))) + await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data))) proc cache*(data: User) {.async.} = if data.username.len == 0: return @@ -145,6 +145,9 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} = elif fetch: result = await getGraphUser(username) await cache(result) + if result.id.len > 0: + await setEx("i:" & result.id, baseCacheTime, result.username) + await cacheUserId(result.username, result.id) proc getCachedUsername*(userId: string): Future[string] {.async.} = let @@ -174,14 +177,14 @@ proc getCachedTweet*(id: string; after=""): Future[Conversation] {.async.} = if not result.isNil and after.len > 0: result.replies = await getReplies(id, after) -proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} = - if name.len == 0: return - let rail = await get("pr:" & toLower(name)) +proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} = + if id.len == 0: return + let rail = await get("pr2:" & toLower(id)) if rail != redisNil: rail.deserialize(PhotoRail) else: - result = await getPhotoRail(name) - await cache(result, name) + result = await getPhotoRail(id) + await cache(result, id) proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} = let list = if id.len == 0: redisNil diff --git a/src/routes/activityspoof.nim b/src/routes/activityspoof.nim index c5ca98e..08f7e04 100644 --- a/src/routes/activityspoof.nim +++ b/src/routes/activityspoof.nim @@ -19,14 +19,15 @@ proc createActivityPubRouter*(cfg: Config) = get "/api/v1/accounts/@id": let id = @"id" - if id.len > 19 or id.any(c => not c.isDigit): + #if id.len > 19 or id.any(c => not c.isDigit): + if not id.allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}): resp Http404, {"Content-Type": "application/json"}, """{"error":"Invalid account ID"}""" - var username = await getCachedUsername(id) - if username.len == 0: - resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}""" + #var username = await getCachedUsername(id) + #if username.len == 0: + #resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}""" - let user = await getCachedUser(username) + let user = await getCachedUser(id) if user.suspended or user.id.len == 0: resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}""" @@ -189,9 +190,8 @@ proc createActivityPubRouter*(cfg: Config) = postJson["edited_at"] = newJNull() postJson["reblog"] = newJNull() if tweet.replyId.len != 0: - let replyUser = await getCachedUser(tweet.replyHandle) postJson["in_reply_to_id"] = %(&"{tweet.replyId}") - postJson["in_reply_to_account_id"] = %replyUser.id + postJson["in_reply_to_account_id"] = %tweet.replyHandle else: postJson["in_reply_to_id"] = newJNull() postJson["in_reply_to_account_id"] = newJNull() diff --git a/src/routes/debug.nim b/src/routes/debug.nim deleted file mode 100644 index 47a8b22..0000000 --- a/src/routes/debug.nim +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -import jester -import router_utils -import ".."/[tokens, types] - -proc createDebugRouter*(cfg: Config) = - router debug: - get "/.tokens": - cond cfg.enableDebug - respJson getPoolJson() - - #get "/.health": - #respJson getAccountPoolHealth() - - #get "/.accounts": - #cond cfg.enableDebug - #respJson getAccountPoolDebug() diff --git a/src/routes/status.nim b/src/routes/status.nim index f7c69bf..3516a19 100644 --- a/src/routes/status.nim +++ b/src/routes/status.nim @@ -5,7 +5,7 @@ import jester, karax/vdom import router_utils import ".."/[types, formatters, api, redis_cache] -import ../views/[general, status, search, mastoapi] +import ../views/[general, status, mastoapi] export json, uri, sequtils, options, sugar, times export router_utils diff --git a/src/routes/timeline.nim b/src/routes/timeline.nim index a8967d9..d589687 100644 --- a/src/routes/timeline.nim +++ b/src/routes/timeline.nim @@ -48,7 +48,7 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false; let rail = skipIf(skipRail or query.kind == media, @[]): - getCachedPhotoRail(name) + getCachedPhotoRail(userId) user = getCachedUser(name) @@ -111,6 +111,7 @@ proc createTimelineRouter*(cfg: Config) = get "/@name/?@tab?/?": cond '.' notin @"name" cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"] + cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_', ','}) cond @"tab" in ["with_replies", "media", "search", "following", "followers", ""] let prefs = cookiePrefs() diff --git a/src/routes/twitter_api.nim b/src/routes/twitter_api.nim index b58924b..5a04022 100644 --- a/src/routes/twitter_api.nim +++ b/src/routes/twitter_api.nim @@ -4,7 +4,7 @@ import json, asyncdispatch, options, uri import times import jester import router_utils -import ".."/[types, api, apiutils, query, consts, redis_cache] +import ".."/[types, api, apiutils, redis_cache] import httpclient, strutils import sequtils @@ -81,60 +81,49 @@ proc getUserTweetsJson*(id: string): Future[JsonNode] {.async.} = result = response -proc searchTimeline*(query: Query; after=""): Future[string] {.async.} = - let q = genQueryParam(query) - var - variables = %*{ - "rawQuery": q, - "count": 20, - "product": "Latest", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false - } - if after.len > 0: - variables["cursor"] = % after - let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures} - result = await fetchRaw(url, Api.search) - proc getUserTweets*(id: string; after=""): Future[string] {.async.} = if id.len == 0: return - let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = userTweetsVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - result = await fetchRaw(graphUserTweets ? params, Api.userTweets) + + let headers = newHttpHeaders() + headers.add("Referer", """https://x.com/$1""" % id) + + let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + result = await fetchRaw(userTweetsUrl(id, cursor), headers) proc getUserReplies*(id: string; after=""): Future[string] {.async.} = if id.len == 0: return - let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = userTweetsVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - result = await fetchRaw(graphUserTweets ? params, Api.userTweetsAndReplies) + + let headers = newHttpHeaders() + headers.add("Referer", """https://x.com/$1/with_replies""" % id) + + let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + result = await fetchRaw(userTweetsAndRepliesUrl(id, cursor), headers) proc getUserMedia*(id: string; after=""): Future[string] {.async.} = if id.len == 0: return - let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = userMediaVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - result = await fetchRaw(graphUserMedia ? params, Api.userMedia) + + let headers = newHttpHeaders() + headers.add("Referer", """https://x.com/$1/media""" % id) + + let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + result = await fetchRaw(mediaUrl(id, cursor), headers) proc getTweetById*(id: string; after=""): Future[string] {.async.} = - let - cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" - variables = tweetVariables % [id, cursor] - params = {"variables": variables, "features": gqlFeatures} - result = await fetchRaw(graphTweet ? params, Api.tweetDetail) + if id.len == 0: return + + let headers = newHttpHeaders() + headers.add("Referer", """https://x.com/i/status/$1""" % id) + + let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: "" + result = await fetchRaw(tweetDetailUrl(id, cursor), headers) proc getUser*(username: string): Future[string] {.async.} = if username.len == 0: return - let - variables = """{"screen_name":"$1"}""" % username - fieldToggles = """{"withAuxiliaryUserLabels":true}""" - params = {"variables": variables, "features": gqlFeatures, "fieldToggles": fieldToggles} - result = await fetchRaw(graphUser ? params, Api.userScreenName) + + let headers = newHttpHeaders() + headers.add("Referer", """https://x.com/$1""" % username) + + result = await fetchRaw(userUrl(username), headers) proc createTwitterApiRouter*(cfg: Config) = router api: @@ -146,16 +135,11 @@ proc createTwitterApiRouter*(cfg: Config) = let response = await getUser(username) resp Http200, { "Content-Type": "application/json" }, response - #get "/api/user/@id/tweets": - # let id = @"id" - # let response = await getUserTweetsJson(id) - # respJson response - - get "/api/user/@username/timeline": - let username = @"username" - let query = Query(fromUser: @[username]) - let response = await searchTimeline(query) - resp Http200, { "Content-Type": "application/json" }, response + #get "/api/user/@username/timeline": + # let username = @"username" + # let query = Query(fromUser: @[username]) + # let response = await searchTimeline(query) + # resp Http200, { "Content-Type": "application/json" }, response get "/api/user/@id/tweets": let id = @"id" diff --git a/src/sass/general.scss b/src/sass/general.scss index debf89f..2064213 100644 --- a/src/sass/general.scss +++ b/src/sass/general.scss @@ -23,6 +23,7 @@ font-weight: bold; width: 30px; height: 30px; + padding: 0px 5px 1px 8px; } input { diff --git a/src/sass/index.scss b/src/sass/index.scss index 6cab48e..94582ce 100644 --- a/src/sass/index.scss +++ b/src/sass/index.scss @@ -1,180 +1,202 @@ -@import '_variables'; +@import "_variables"; -@import 'tweet/_base'; -@import 'profile/_base'; -@import 'general'; -@import 'navbar'; -@import 'inputs'; -@import 'timeline'; -@import 'search'; +@import "tweet/_base"; +@import "profile/_base"; +@import "general"; +@import "navbar"; +@import "inputs"; +@import "timeline"; +@import "search"; body { - // colors - --bg_color: #{$bg_color}; - --fg_color: #{$fg_color}; - --fg_faded: #{$fg_faded}; - --fg_dark: #{$fg_dark}; - --fg_nav: #{$fg_nav}; + // colors + --bg_color: #{$bg_color}; + --fg_color: #{$fg_color}; + --fg_faded: #{$fg_faded}; + --fg_dark: #{$fg_dark}; + --fg_nav: #{$fg_nav}; - --bg_panel: #{$bg_panel}; - --bg_elements: #{$bg_elements}; - --bg_overlays: #{$bg_overlays}; - --bg_hover: #{$bg_hover}; + --bg_panel: #{$bg_panel}; + --bg_elements: #{$bg_elements}; + --bg_overlays: #{$bg_overlays}; + --bg_hover: #{$bg_hover}; - --grey: #{$grey}; - --dark_grey: #{$dark_grey}; - --darker_grey: #{$darker_grey}; - --darkest_grey: #{$darkest_grey}; - --border_grey: #{$border_grey}; + --grey: #{$grey}; + --dark_grey: #{$dark_grey}; + --darker_grey: #{$darker_grey}; + --darkest_grey: #{$darkest_grey}; + --border_grey: #{$border_grey}; - --accent: #{$accent}; - --accent_light: #{$accent_light}; - --accent_dark: #{$accent_dark}; - --accent_border: #{$accent_border}; + --accent: #{$accent}; + --accent_light: #{$accent_light}; + --accent_dark: #{$accent_dark}; + --accent_border: #{$accent_border}; - --play_button: #{$play_button}; - --play_button_hover: #{$play_button_hover}; + --play_button: #{$play_button}; + --play_button_hover: #{$play_button_hover}; - --more_replies_dots: #{$more_replies_dots}; - --error_red: #{$error_red}; + --more_replies_dots: #{$more_replies_dots}; + --error_red: #{$error_red}; - --verified_blue: #{$verified_blue}; - --verified_business: #{$verified_business}; - --verified_government: #{$verified_government}; - --icon_text: #{$icon_text}; + --verified_blue: #{$verified_blue}; + --verified_business: #{$verified_business}; + --verified_government: #{$verified_government}; + --icon_text: #{$icon_text}; - --tab: #{$fg_color}; - --tab_selected: #{$accent}; + --tab: #{$fg_color}; + --tab_selected: #{$accent}; - --profile_stat: #{$fg_color}; + --profile_stat: #{$fg_color}; - background-color: var(--bg_color); - color: var(--fg_color); - font-family: $font_0, $font_1, $font_2, $font_3; - font-size: 14px; - line-height: 1.3; - margin: 0; + background-color: var(--bg_color); + color: var(--fg_color); + font-family: $font_0, $font_1; + font-size: 14px; + line-height: 1.3; + margin: 0; } * { - outline: unset; - margin: 0; - text-decoration: none; + outline: unset; + margin: 0; + text-decoration: none; } h1 { - display: inline; + display: inline; } -h2, h3 { - font-weight: normal; +h2, +h3 { + font-weight: normal; } p { - margin: 14px 0; + margin: 14px 0; } a { - color: var(--accent); + color: var(--accent); - &:hover { - text-decoration: underline; - } + &:hover { + text-decoration: underline; + } } fieldset { - border: 0; - padding: 0; - margin-top: -0.6em; + border: 0; + padding: 0; + margin-top: -0.6em; } legend { - width: 100%; - padding: .6em 0 .3em 0; - border: 0; - font-size: 16px; - font-weight: 600; - border-bottom: 1px solid var(--border_grey); - margin-bottom: 8px; + width: 100%; + padding: 0.6em 0 0.3em 0; + border: 0; + font-size: 16px; + font-weight: 600; + border-bottom: 1px solid var(--border_grey); + margin-bottom: 8px; } .preferences .note { - border-top: 1px solid var(--border_grey); - border-bottom: 1px solid var(--border_grey); - padding: 6px 0 8px 0; - margin-bottom: 8px; - margin-top: 16px; + border-top: 1px solid var(--border_grey); + border-bottom: 1px solid var(--border_grey); + padding: 6px 0 8px 0; + margin-bottom: 8px; + margin-top: 16px; } ul { - padding-left: 1.3em; + padding-left: 1.3em; } .container { - display: flex; - flex-wrap: wrap; - box-sizing: border-box; - padding-top: 50px; - margin: auto; - min-height: 100vh; + display: flex; + flex-wrap: wrap; + box-sizing: border-box; + padding-top: 50px; + margin: auto; + min-height: 100vh; } .icon-container { - display: inline; + display: inline; } .overlay-panel { - max-width: 600px; - width: 100%; - margin: 0 auto; - margin-top: 10px; - background-color: var(--bg_overlays); - padding: 10px 15px; - align-self: start; + max-width: 600px; + width: 100%; + margin: 0 auto; + margin-top: 10px; + background-color: var(--bg_overlays); + padding: 10px 15px; + align-self: start; - ul { - margin-bottom: 14px; - } + ul { + margin-bottom: 14px; + } - p { - word-break: break-word; - } + p { + word-break: break-word; + } } .verified-icon { - color: var(--icon_text); - border-radius: 50%; - flex-shrink: 0; - margin: 2px 0 3px 3px; - padding-top: 3px; - height: 11px; - width: 14px; - font-size: 8px; - display: inline-block; - text-align: center; - vertical-align: middle; + display: inline-block; + width: 14px; + height: 14px; + margin-left: 2px; - &.blue { - background-color: var(--verified_blue); + .verified-icon-circle { + position: absolute; + font-size: 15px; + } + + .verified-icon-check { + position: absolute; + font-size: 9px; + margin: 5px 3px; + } + + &.blue { + .verified-icon-circle { + color: var(--verified_blue); } - &.business { - color: var(--bg_panel); - background-color: var(--verified_business); + .verified-icon-check { + color: var(--icon_text); + } + } + + &.business { + .verified-icon-circle { + color: var(--verified_business); } - &.government { - color: var(--bg_panel); - background-color: var(--verified_government); + .verified-icon-check { + color: var(--bg_panel); } + } + + &.government { + .verified-icon-circle { + color: var(--verified_government); + } + + .verified-icon-check { + color: var(--bg_panel); + } + } } -@media(max-width: 600px) { - .preferences-container { - max-width: 95vw; - } +@media (max-width: 600px) { + .preferences-container { + max-width: 95vw; + } - .nav-item, .nav-item .icon-container { - font-size: 16px; - } + .nav-item, + .nav-item .icon-container { + font-size: 16px; + } } diff --git a/src/sass/navbar.scss b/src/sass/navbar.scss index 47a8765..86bfbe7 100644 --- a/src/sass/navbar.scss +++ b/src/sass/navbar.scss @@ -1,89 +1,87 @@ -@import '_variables'; +@import "_variables"; nav { - display: flex; - align-items: center; - position: fixed; - background-color: var(--bg_overlays); - box-shadow: 0 0 4px $shadow; - padding: 0; - width: 100%; - height: 50px; - z-index: 1000; - font-size: 16px; + display: flex; + align-items: center; + position: fixed; + background-color: var(--bg_overlays); + box-shadow: 0 0 4px $shadow; + padding: 0; + width: 100%; + height: 50px; + z-index: 1000; + font-size: 16px; - a, .icon-button button { - color: var(--fg_nav); - } + a, + .icon-button button { + color: var(--fg_nav); + } } .inner-nav { - margin: auto; - box-sizing: border-box; - padding: 0 10px; - display: flex; - align-items: center; - flex-basis: 920px; - height: 50px; + margin: auto; + box-sizing: border-box; + padding: 0 10px; + display: flex; + align-items: center; + flex-basis: 920px; + height: 50px; } .site-name { - font-size: 15px; - font-weight: 600; - line-height: 1; + font-size: 15px; + font-weight: 600; + line-height: 1; - &:hover { - color: var(--accent_light); - text-decoration: unset; - } + &:hover { + color: var(--accent_light); + text-decoration: unset; + } } .site-logo { - display: block; - width: 35px; - height: 35px; + display: block; + width: 35px; + height: 35px; } .nav-item { - display: flex; - flex: 1; - line-height: 50px; - height: 50px; - overflow: hidden; - flex-wrap: wrap; - align-items: center; + display: flex; + flex: 1; + line-height: 50px; + height: 50px; + overflow: hidden; + flex-wrap: wrap; + align-items: center; - &.right { - text-align: right; - justify-content: flex-end; - } + &.right { + text-align: right; + justify-content: flex-end; + } - &.right a { - padding-left: 4px; - - &:hover { - color: var(--accent_light); - text-decoration: unset; - } - } + &.right a:hover { + color: var(--accent_light); + text-decoration: unset; + } } .lp { - height: 14px; - display: inline-block; - position: relative; - top: 2px; - fill: var(--fg_nav); + height: 14px; + display: inline-block; + position: relative; + top: 2px; + fill: var(--fg_nav); - &:hover { - fill: var(--accent_light); - } + &:hover { + fill: var(--accent_light); + } } -.icon-info:before { - margin: 0 -3px; +.icon-info { + margin: 0 -3px; } .icon-cog { - font-size: 15px; + font-size: 15px; + padding-left: 0 !important; } diff --git a/src/sass/search.scss b/src/sass/search.scss index f70f7ea..458290f 100644 --- a/src/sass/search.scss +++ b/src/sass/search.scss @@ -1,120 +1,121 @@ -@import '_variables'; -@import '_mixins'; +@import "_variables"; +@import "_mixins"; .search-title { - font-weight: bold; - display: inline-block; - margin-top: 4px; + font-weight: bold; + display: inline-block; + margin-top: 4px; } .search-field { + display: flex; + flex-wrap: wrap; + + button { + margin: 0 2px 0 0; + padding: 0px 1px 1px 4px; + height: 23px; display: flex; - flex-wrap: wrap; + align-items: center; + } - button { - margin: 0 2px 0 0; - height: 23px; - display: flex; - align-items: center; - } + .pref-input { + margin: 0 4px 0 0; + flex-grow: 1; + height: 23px; + } - .pref-input { - margin: 0 4px 0 0; - flex-grow: 1; - height: 23px; - } + input[type="text"] { + height: calc(100% - 4px); + width: calc(100% - 8px); + } - input[type="text"] { - height: calc(100% - 4px); - width: calc(100% - 8px); - } + > label { + display: inline; + background-color: var(--bg_elements); + color: var(--fg_color); + border: 1px solid var(--accent_border); + padding: 1px 1px 2px 4px; + font-size: 14px; + cursor: pointer; + margin-bottom: 2px; - > label { - display: inline; - background-color: var(--bg_elements); - color: var(--fg_color); - border: 1px solid var(--accent_border); - padding: 1px 6px 2px 6px; - font-size: 14px; - cursor: pointer; - margin-bottom: 2px; + @include input-colors; + } - @include input-colors; - } - - @include create-toggle(search-panel, 200px); + @include create-toggle(search-panel, 200px); } .search-panel { - width: 100%; - max-height: 0; - overflow: hidden; - transition: max-height 0.4s; + width: 100%; + max-height: 0; + overflow: hidden; + transition: max-height 0.4s; - flex-grow: 1; - font-weight: initial; - text-align: left; + flex-grow: 1; + font-weight: initial; + text-align: left; - > div { - line-height: 1.7em; - } + > div { + line-height: 1.7em; + } - .checkbox-container { - display: inline; - padding-right: unset; - margin-bottom: unset; - margin-left: 23px; - } + .checkbox-container { + display: inline; + padding-right: unset; + margin-bottom: 5px; + margin-left: 23px; + } - .checkbox { - right: unset; - left: -22px; - } + .checkbox { + right: unset; + left: -22px; + } - .checkbox-container .checkbox:after { - top: -4px; - } + .checkbox-container .checkbox:after { + top: -4px; + } } .search-row { - display: flex; - flex-wrap: wrap; - line-height: unset; + display: flex; + flex-wrap: wrap; + line-height: unset; - > div { - flex-grow: 1; - flex-shrink: 1; - } + > div { + flex-grow: 1; + flex-shrink: 1; + } + + input { + height: 21px; + } + + .pref-input { + display: block; + padding-bottom: 5px; input { - height: 21px; - } - - .pref-input { - display: block; - padding-bottom: 5px; - - input { - height: 21px; - margin-top: 1px; - } + height: 21px; + margin-top: 1px; } + } } .search-toggles { - flex-grow: 1; - display: grid; - grid-template-columns: repeat(6, auto); - grid-column-gap: 10px; + flex-grow: 1; + display: grid; + grid-template-columns: repeat(6, auto); + grid-column-gap: 10px; } .profile-tabs { - @include search-resize(820px, 5); - @include search-resize(725px, 4); - @include search-resize(600px, 6); - @include search-resize(560px, 5); - @include search-resize(480px, 4); - @include search-resize(410px, 3); + @include search-resize(820px, 5); + @include search-resize(725px, 4); + @include search-resize(600px, 6); + @include search-resize(560px, 5); + @include search-resize(480px, 4); + @include search-resize(410px, 3); } @include search-resize(560px, 5); diff --git a/src/sass/tweet/video.scss b/src/sass/tweet/video.scss index 1e9096e..37bab98 100644 --- a/src/sass/tweet/video.scss +++ b/src/sass/tweet/video.scss @@ -1,66 +1,64 @@ -@import '_variables'; -@import '_mixins'; +@import "_variables"; +@import "_mixins"; video { - max-height: 100%; - width: 100%; + height: 100%; + width: 100%; } .gallery-video { - display: flex; - overflow: hidden; + display: flex; + overflow: hidden; } .gallery-video.card-container { - flex-direction: column; + flex-direction: column; + width: 100%; } .video-container { - max-height: 530px; - margin: 0; - display: flex; - align-items: center; - justify-content: center; + max-height: 530px; + margin: 0; - img { - max-height: 100%; - max-width: 100%; - } + img { + max-height: 100%; + max-width: 100%; + } } .video-overlay { - @include play-button; - background-color: $shadow; + @include play-button; + background-color: $shadow; - p { - position: relative; - z-index: 0; - text-align: center; - top: calc(50% - 20px); - font-size: 20px; - line-height: 1.3; - margin: 0 20px; - } + p { + position: relative; + z-index: 0; + text-align: center; + top: calc(50% - 20px); + font-size: 20px; + line-height: 1.3; + margin: 0 20px; + } - div { - position: relative; - z-index: 0; - top: calc(50% - 20px); - margin: 0 auto; - width: 40px; - height: 40px; - } + div { + position: relative; + z-index: 0; + top: calc(50% - 20px); + margin: 0 auto; + width: 40px; + height: 40px; + } - form { - width: 100%; - height: 100%; - align-items: center; - justify-content: center; - display: flex; - } + form { + width: 100%; + height: 100%; + align-items: center; + justify-content: center; + display: flex; + } - button { - padding: 5px 8px; - font-size: 16px; - } + button { + padding: 5px 8px; + font-size: 16px; + } } diff --git a/src/tid.nim b/src/tid.nim new file mode 100644 index 0000000..7b453fb --- /dev/null +++ b/src/tid.nim @@ -0,0 +1,62 @@ +import std/[asyncdispatch, base64, httpclient, random, strutils, sequtils, times] +import nimcrypto +import experimental/parser/tid + +randomize() + +const defaultKeyword = "obfiowerehiring"; +const pairsUrl = + "https://raw.githubusercontent.com/fa0311/x-client-transaction-id-pair-dict/refs/heads/main/pair.json"; + +var + cachedPairs: seq[TidPair] = @[] + lastCached = 0 + # refresh every hour + ttlSec = 60 * 60 + +proc getPair(): Future[TidPair] {.async.} = + if cachedPairs.len == 0 or int(epochTime()) - lastCached > ttlSec: + lastCached = int(epochTime()) + + let client = newAsyncHttpClient() + defer: client.close() + + let resp = await client.get(pairsUrl) + if resp.status == $Http200: + cachedPairs = parseTidPairs(await resp.body) + + return sample(cachedPairs) + +proc encodeSha256(text: string): array[32, byte] = + let + data = cast[ptr byte](addr text[0]) + dataLen = uint(len(text)) + digest = sha256.digest(data, dataLen) + return digest.data + +proc encodeBase64[T](data: T): string = + return encode(data).replace("=", "") + +proc decodeBase64(data: string): seq[byte] = + return cast[seq[byte]](decode(data)) + +proc genTid*(path: string): Future[string] {.async.} = + let + pair = await getPair() + + timeNow = int(epochTime() - 1682924400) + timeNowBytes = @[ + byte(timeNow and 0xff), + byte((timeNow shr 8) and 0xff), + byte((timeNow shr 16) and 0xff), + byte((timeNow shr 24) and 0xff) + ] + + data = "GET!" & path & "!" & $timeNow & defaultKeyword & pair.animationKey + hashBytes = encodeSha256(data) + keyBytes = decodeBase64(pair.verification) + bytesArr = keyBytes & timeNowBytes & hashBytes[0 ..< 16] & @[3'u8] + randomNum = byte(rand(256)) + tid = @[randomNum] & bytesArr.mapIt(it xor randomNum) + + return encodeBase64(tid) diff --git a/src/tokens.nim b/src/tokens.nim deleted file mode 100644 index f622e4d..0000000 --- a/src/tokens.nim +++ /dev/null @@ -1,168 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -import asyncdispatch, httpclient, times, sequtils, json, random -import strutils, tables -import types, consts - -const - maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions - maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires - maxAge = 2.hours + 55.minutes # tokens expire after 3 hours - failDelay = initDuration(minutes=30) - -var - tokenPool: seq[Token] - lastFailed: Time - enableLogging = false - -let headers = newHttpHeaders({"authorization": auth}) - -template log(str) = - if enableLogging: echo "[tokens] ", str - -proc getPoolJson*(): JsonNode = - var - list = newJObject() - totalReqs = 0 - totalPending = 0 - reqsPerApi: Table[string, int] - - for token in tokenPool: - totalPending.inc(token.pending) - list[token.tok] = %*{ - "apis": newJObject(), - "pending": token.pending, - "init": $token.init, - "lastUse": $token.lastUse - } - - for api in token.apis.keys: - list[token.tok]["apis"][$api] = %token.apis[api] - - let - maxReqs = - case api - of Api.photoRail: 180 - #of Api.timeline: 187 - #of Api.userTweets, Api.userTimeline: 300 - of Api.userTweets: 300 - of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets, - Api.userTweetsAndReplies, Api.userMedia, - Api.userRestId, Api.userScreenName, Api.tweetDetail, - Api.tweetResult, Api.search, Api.favorites, - Api.retweeters, Api.favoriters, Api.following, Api.followers: 500 - #of Api.userSearch: 900 - reqs = maxReqs - token.apis[api].remaining - - reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs - totalReqs.inc(reqs) - - return %*{ - "amount": tokenPool.len, - "requests": totalReqs, - "pending": totalPending, - "apis": reqsPerApi, - "tokens": list - } - -proc rateLimitError*(): ref RateLimitError = - newException(RateLimitError, "rate limited") - -proc fetchToken(): Future[Token] {.async.} = - if getTime() - lastFailed < failDelay: - raise rateLimitError() - - let client = newAsyncHttpClient(headers=headers) - - try: - let - resp = await client.postContent(activate) - tokNode = parseJson(resp)["guest_token"] - tok = tokNode.getStr($(tokNode.getInt)) - time = getTime() - - return Token(tok: tok, init: time, lastUse: time) - except Exception as e: - echo "[tokens] fetching token failed: ", e.msg - if "Try again" notin e.msg: - echo "[tokens] fetching tokens paused, resuming in 30 minutes" - lastFailed = getTime() - finally: - client.close() - -proc expired(token: Token): bool = - let time = getTime() - token.init < time - maxAge or token.lastUse < time - maxLastUse - -proc isLimited(token: Token; api: Api): bool = - if token.isNil or token.expired: - return true - - if api in token.apis: - let limit = token.apis[api] - return (limit.remaining <= 10 and limit.reset > epochTime().int) - else: - return false - -proc isReady(token: Token; api: Api): bool = - not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api)) - -proc release*(token: Token; used=false; invalid=false) = - if token.isNil: return - if invalid or token.expired: - if invalid: log "discarding invalid token" - elif token.expired: log "discarding expired token" - - let idx = tokenPool.find(token) - if idx > -1: tokenPool.delete(idx) - elif used: - dec token.pending - token.lastUse = getTime() - -proc getToken*(api: Api): Future[Token] {.async.} = - for i in 0 ..< tokenPool.len: - if result.isReady(api): break - release(result) - result = tokenPool.sample() - - if not result.isReady(api): - release(result) - result = await fetchToken() - log "added new token to pool" - tokenPool.add result - - if not result.isNil: - inc result.pending - else: - raise rateLimitError() - -proc setRateLimit*(token: Token; api: Api; remaining, reset: int) = - # avoid undefined behavior in race conditions - if api in token.apis: - let limit = token.apis[api] - if limit.reset >= reset and limit.remaining < remaining: - return - - token.apis[api] = RateLimit(remaining: remaining, reset: reset) - -proc poolTokens*(amount: int) {.async.} = - var futs: seq[Future[Token]] - for i in 0 ..< amount: - futs.add fetchToken() - - for token in futs: - var newToken: Token - - try: newToken = await token - except: discard - - if not newToken.isNil: - log "added new token to pool" - tokenPool.add newToken - -proc initTokenPool*(cfg: Config) {.async.} = - enableLogging = cfg.enableDebug - - while true: - if tokenPool.countIt(not it.isLimited(Api.userTweets)) < cfg.minTokens: - await poolTokens(min(4, cfg.minTokens - tokenPool.len)) - await sleepAsync(2000) \ No newline at end of file diff --git a/src/types.nim b/src/types.nim index 38df172..34e61e8 100644 --- a/src/types.nim +++ b/src/types.nim @@ -12,25 +12,13 @@ type TimelineKind* {.pure.} = enum tweets, replies, media - Api* {.pure.} = enum - tweetDetail - tweetResult - photoRail - search - list - listBySlug - listMembers - listTweets - userRestId - userScreenName - favorites - userTweets - userTweetsAndReplies - userMedia - favoriters - retweeters - following - followers + ApiUrl* = object + endpoint*: string + params*: seq[(string, string)] + + ApiReq* = object + oauth*: ApiUrl + cookie*: ApiUrl RateLimit* = object remaining*: int @@ -43,14 +31,14 @@ type init*: Time lastUse*: Time pending*: int - apis*: Table[Api, RateLimit] + apis*: Table[string, RateLimit] GuestAccount* = ref object id*: int64 oauthToken*: string oauthSecret*: string pending*: int - apis*: Table[Api, RateLimit] + apis*: Table[string, RateLimit] Error* = enum null = 0 @@ -308,6 +296,7 @@ type enableDebug*: bool proxy*: string proxyAuth*: string + disableTid*: bool cookieHeader*: string xCsrfToken*: string diff --git a/src/views/renderutils.nim b/src/views/renderutils.nim index f1bbb80..7b29bb2 100644 --- a/src/views/renderutils.nim +++ b/src/views/renderutils.nim @@ -26,7 +26,9 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode = template verifiedIcon*(user: User): untyped {.dirty.} = if user.verifiedType != VerifiedType.none: let lower = ($user.verifiedType).toLowerAscii() - icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account") + buildHtml(tdiv(class=(&"verified-icon {lower}"))): + icon "circle", class="verified-icon-circle", title=(&"Verified {lower} account") + icon "ok", class="verified-icon-check", title=(&"Verified {lower} account") else: text "" diff --git a/src/views/rss.nimf b/src/views/rss.nimf index cdfcd63..27c2f4e 100644 --- a/src/views/rss.nimf +++ b/src/views/rss.nimf @@ -2,6 +2,9 @@ ## SPDX-License-Identifier: AGPL-3.0-only #import strutils, xmltree, strformat, options, unicode #import ../types, ../utils, ../formatters, ../prefs +## Snowflake ID cutoff for RSS GUID format transition +## Corresponds to approximately December 14, 2025 UTC +#const guidCutoff = 2000000000000000000'i64 # #proc getTitle(tweet: Tweet; retweet: string): string = #if tweet.pinned: result = "Pinned: " @@ -25,7 +28,25 @@ #end proc # #proc getDescription(desc: string; cfg: Config): string = -Twitter feed for: ${desc}. Generated by ${cfg.hostname} +Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)} +#end proc +# +#proc getTweetsWithPinned(profile: Profile): seq[Tweets] = +#result = profile.tweets.content +#if profile.pinned.isSome and result.len > 0: +# let pinnedTweet = profile.pinned.get +# var inserted = false +# for threadIdx in 0 ..< result.len: +# if not inserted: +# for tweetIdx in 0 ..< result[threadIdx].len: +# if result[threadIdx][tweetIdx].id < pinnedTweet.id: +# result[threadIdx].insert(pinnedTweet, tweetIdx) +# inserted = true +# end if +# end for +# end if +# end for +#end if #end proc # #proc renderRssTweet(tweet: Tweet; cfg: Config): string = @@ -38,23 +59,34 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname} # end for #elif tweet.video.isSome: - + +
Video
+ + #elif tweet.gif.isSome: # let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}" # let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}" -##elif tweet.card.isSome: -## let card = tweet.card.get() -## if card.image.len > 0: -## -## end if +#elif tweet.card.isSome: +# let card = tweet.card.get() +# if card.image.len > 0: + +# end if #end if #if tweet.quote.isSome and get(tweet.quote).available: -# let quoteLink = getLink(get(tweet.quote)) +# let quoteTweet = get(tweet.quote) +# let quoteLink = urlPrefix & getLink(quoteTweet)
-

Quoting: ${cfg.hostname}${quoteLink}

-${renderRssTweet(get(tweet.quote), cfg)} +
+${quoteTweet.user.fullname} (@${quoteTweet.user.username}) +

+${renderRssTweet(quoteTweet, cfg)} +

+ +
#end if #end proc # @@ -72,12 +104,17 @@ ${renderRssTweet(get(tweet.quote), cfg)} # if link in links: continue # end if # links.add link +# let useGlobalGuid = parseBiggestInt(tweet.id) >= guidCutoff ${getTitle(tweet, retweet)} @${tweet.user.username} ${getRfc822Time(tweet)} +#if useGlobalGuid: + ${tweet.id} +#else: ${urlPrefix & link} +#end if ${urlPrefix & link} # end for @@ -108,8 +145,9 @@ ${renderRssTweet(get(tweet.quote), cfg)} 128 128 -#if profile.tweets.content.len > 0: -${renderRssTweets(profile.tweets.content, cfg, userId=profile.user.id)} +#let tweetsList = getTweetsWithPinned(profile) +#if tweetsList.len > 0: +${renderRssTweets(tweetsList, cfg, userId=profile.user.id)} #end if diff --git a/src/views/status.nim b/src/views/status.nim index c3adb3d..090604e 100644 --- a/src/views/status.nim +++ b/src/views/status.nim @@ -28,14 +28,19 @@ proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode = if thread.hasMore: renderMoreReplies(thread) -proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode = +proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string; tweet: Tweet = nil): VNode = buildHtml(tdiv(class="replies", id="r")): + var hasReplies = false + var replyCount = 0 for thread in replies.content: if thread.content.len == 0: continue + hasReplies = true + replyCount += thread.content.len renderReplyThread(thread, prefs, path) - if replies.bottom.len > 0: - renderMore(Query(), replies.bottom, focus="#r") + if hasReplies and replies.bottom.len > 0: + if tweet == nil or not replies.beginning or replyCount < tweet.stats.replies: + renderMore(Query(), replies.bottom, focus="#r") proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode = let hasAfter = conv.after.content.len > 0 @@ -70,6 +75,6 @@ proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode if not conv.replies.beginning: renderNewer(Query(), getLink(conv.tweet), focus="#r") if conv.replies.content.len > 0 or conv.replies.bottom.len > 0: - renderReplies(conv.replies, prefs, path) + renderReplies(conv.replies, prefs, path, conv.tweet) renderToTop(focus="#m")