bunch of upstream changes
This commit is contained in:
Cynthia Foxwell
parent
116652c2a5
commit
0bc6b33251
@ -26,12 +26,7 @@ enableRSS = true # set this to false to disable RSS feeds
|
||||
enableDebug = false # enable request logs and debug endpoints (/.accounts)
|
||||
proxy = "" # http/https url, SOCKS proxies are not supported
|
||||
proxyAuth = ""
|
||||
tokenCount = 10
|
||||
# minimum amount of usable tokens. tokens are used to authorize API requests,
|
||||
# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
|
||||
# the limits reset every 15 minutes, and the pool is filled up so there's
|
||||
# always at least `tokenCount` usable tokens. only increase this if you receive
|
||||
# major bursts all the time and don't have a rate limiting setup via e.g. nginx
|
||||
disableTid = false # enable this if cookie-based auth is failing
|
||||
|
||||
# Change default preferences here, see src/prefs_impl.nim for a complete list
|
||||
[Preferences]
|
||||
|
||||
185
src/api.nim
185
src/api.nim
@ -1,78 +1,114 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import asyncdispatch, httpclient, uri, strutils, sequtils, sugar
|
||||
import asyncdispatch, httpclient, strutils, sequtils, sugar
|
||||
import packedjson
|
||||
import types, query, formatters, consts, apiutils, parser
|
||||
import experimental/parser as newParser
|
||||
|
||||
# Helper to generate params object for GraphQL requests
|
||||
proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
|
||||
result.add ("variables", variables)
|
||||
result.add ("features", gqlFeatures)
|
||||
if fieldToggles.len > 0:
|
||||
result.add ("fieldToggles", fieldToggles)
|
||||
|
||||
proc apiUrl*(endpoint, variables: string; fieldToggles = ""): ApiUrl =
|
||||
return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
|
||||
|
||||
proc apiReq*(endpoint, variables: string; fieldToggles = ""): ApiReq =
|
||||
let url = apiUrl(endpoint, variables, fieldToggles)
|
||||
return ApiReq(cookie: url, oauth: url)
|
||||
|
||||
proc mediaUrl*(id: string; cursor: string): ApiReq =
|
||||
result = ApiReq(
|
||||
cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor], """{"withArticlePlainText":false}"""),
|
||||
oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor])
|
||||
)
|
||||
|
||||
proc userTweetsUrl*(id: string; cursor: string): ApiReq =
|
||||
result = ApiReq(
|
||||
cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
|
||||
oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor])
|
||||
)
|
||||
|
||||
proc userTweetsAndRepliesUrl*(id: string; cursor: string): ApiReq =
|
||||
let cookieVars = userTweetsAndRepliesVars % [id, cursor]
|
||||
result = ApiReq(
|
||||
cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
|
||||
oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor])
|
||||
)
|
||||
|
||||
proc tweetDetailUrl*(id: string; cursor: string): ApiReq =
|
||||
result = ApiReq(
|
||||
cookie: apiUrl(graphTweetDetail, tweetDetailVars % [id, cursor], tweetDetailFieldToggles),
|
||||
oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
|
||||
)
|
||||
|
||||
proc userUrl*(username: string): ApiReq =
|
||||
let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
|
||||
result = ApiReq(
|
||||
cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
|
||||
oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
|
||||
)
|
||||
|
||||
proc getGraphUser*(username: string): Future[User] {.async.} =
|
||||
if username.len == 0: return
|
||||
|
||||
let
|
||||
headers = newHttpHeaders()
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/$1""" % username)
|
||||
|
||||
let
|
||||
variables = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
|
||||
fieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
|
||||
params = {"variables": variables, "features": userFeatures, "fieldToggles": fieldToggles}
|
||||
js = await fetch(graphUser ? params, Api.userScreenName, headers)
|
||||
let js = await fetchRaw(userUrl(username), headers)
|
||||
result = parseGraphUser(js)
|
||||
|
||||
proc getGraphUserById*(id: string): Future[User] {.async.} =
|
||||
if id.len == 0 or id.any(c => not c.isDigit): return
|
||||
let
|
||||
headers = newHttpHeaders()
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/i/user/$1""" % id)
|
||||
|
||||
let
|
||||
variables = """{"userId":"$1"}""" % id
|
||||
params = {"variables": variables, "features": userFeatures}
|
||||
js = await fetch(graphUserById ? params, Api.userRestId, headers)
|
||||
url = apiReq(graphUserById, """{"userId":"$1"}""" % id)
|
||||
js = await fetchRaw(url, headers)
|
||||
result = parseGraphUser(js)
|
||||
|
||||
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
|
||||
if id.len == 0: return
|
||||
|
||||
let endpoint = case kind
|
||||
let
|
||||
endpoint = case kind
|
||||
of TimelineKind.tweets: ""
|
||||
of TimelineKind.replies: "/with_replies"
|
||||
of TimelineKind.media: "/media"
|
||||
let
|
||||
headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/$1$2""" % [id, endpoint])
|
||||
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = if kind == TimelineKind.media: userMediaVariables % [id, cursor] else: userTweetsVariables % [id, cursor]
|
||||
fieldToggles = """{"withArticlePlainText":true}"""
|
||||
params = {"variables": variables, "features": gqlFeatures, "fieldToggles": fieldToggles}
|
||||
(url, apiId) = case kind
|
||||
of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
|
||||
of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
|
||||
of TimelineKind.media: (graphUserMedia, Api.userMedia)
|
||||
js = await fetch(url ? params, apiId, headers)
|
||||
url = case kind
|
||||
of TimelineKind.tweets: userTweetsUrl(id, cursor)
|
||||
of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
|
||||
of TimelineKind.media: mediaUrl(id, cursor)
|
||||
js = await fetch(url, headers)
|
||||
result = parseGraphTimeline(js, if kind == TimelineKind.media: "" else: "user", after)
|
||||
|
||||
proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = listTweetsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphListTweets ? params, Api.listTweets)
|
||||
url = apiReq(graphListTweets, restIdVars % [id, cursor])
|
||||
js = await fetch(url)
|
||||
result = parseGraphTimeline(js, "list", after).tweets
|
||||
|
||||
proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
|
||||
let
|
||||
variables = %*{"screenName": name, "listSlug": list}
|
||||
params = {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
|
||||
url = apiReq(graphListBySlug, $variables)
|
||||
js = await fetch(url)
|
||||
result = parseGraphList(js)
|
||||
|
||||
proc getGraphList*(id: string): Future[List] {.async.} =
|
||||
let
|
||||
variables = """{"listId":"$1"}""" % id
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
result = parseGraphList(await fetch(graphListById ? params, Api.list))
|
||||
url = apiReq(graphListById, """{"listId": "$1"}""" % id)
|
||||
js = await fetch(url)
|
||||
result = parseGraphList(js)
|
||||
|
||||
proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
|
||||
if list.id.len == 0: return
|
||||
@ -86,85 +122,45 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
|
||||
|
||||
proc getFavorites*(id: string; cfg: Config; after=""): Future[Profile] {.async.} =
|
||||
if id.len == 0: return
|
||||
var
|
||||
variables = %*{
|
||||
"userId": id,
|
||||
"includePromotedContent":false,
|
||||
"withClientEventToken":false,
|
||||
"withBirdwatchNotes":true,
|
||||
"withVoice":true,
|
||||
"withV2Timeline":false
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
let
|
||||
url = consts.favorites ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphTimeline(await fetch(url, Api.favorites), after)
|
||||
url = apiReq(graphListMembers, $variables)
|
||||
js = await fetchRaw(url)
|
||||
result = parseGraphListMembers(js, after)
|
||||
|
||||
proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
headers = newHttpHeaders()
|
||||
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/i/status/$1""" % id)
|
||||
|
||||
let
|
||||
variables = """{"rest_id":"$1"}""" % id
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphTweetResult ? params, Api.tweetResult, headers)
|
||||
url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
|
||||
js = await fetch(url, headers)
|
||||
result = parseGraphTweetResult(js)
|
||||
|
||||
proc getGraphTweet*(id: string; after=""): Future[Conversation] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
headers = newHttpHeaders()
|
||||
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/i/status/$1""" % id)
|
||||
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = tweetVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures, "fieldToggles": tweetFieldToggles}
|
||||
js = await fetch(graphTweet ? params, Api.tweetDetail, headers)
|
||||
js = await fetch(tweetDetailUrl(id, cursor), headers)
|
||||
result = parseGraphConversation(js, id)
|
||||
|
||||
proc getGraphFavoriters*(id: string; after=""): Future[UsersTimeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = reactorsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphFavoriters ? params, Api.favoriters)
|
||||
result = parseGraphFavoritersTimeline(js, id)
|
||||
|
||||
proc getGraphRetweeters*(id: string; after=""): Future[UsersTimeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = reactorsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphRetweeters ? params, Api.retweeters)
|
||||
result = parseGraphRetweetersTimeline(js, id)
|
||||
|
||||
proc getGraphFollowing*(id: string; after=""): Future[UsersTimeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = followVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphFollowing ? params, Api.following)
|
||||
js = await fetch(apiReq(graphFollowing, followVars % [id, cursor]))
|
||||
result = parseGraphFollowTimeline(js, id)
|
||||
|
||||
proc getGraphFollowers*(id: string; after=""): Future[UsersTimeline] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = followVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
js = await fetch(graphFollowers ? params, Api.followers)
|
||||
js = await fetch(apiReq(graphFollowers, followVars % [id, cursor]))
|
||||
result = parseGraphFollowTimeline(js, id)
|
||||
|
||||
proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
|
||||
@ -191,8 +187,10 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
|
||||
let
|
||||
url = apiReq(graphSearchTimeline, $variables)
|
||||
js = await fetch(url)
|
||||
result = parseGraphSearch[Tweets](js, after)
|
||||
result.query = query
|
||||
|
||||
proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
|
||||
@ -211,17 +209,16 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
|
||||
variables["cursor"] = % after
|
||||
result.beginning = false
|
||||
|
||||
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = parseGraphSearch[User](await fetch(url, Api.search), after)
|
||||
let
|
||||
url = apiReq(graphSearchTimeline, $variables)
|
||||
js = await fetch(url)
|
||||
result = parseGraphSearch[User](js, after)
|
||||
result.query = query
|
||||
|
||||
proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
|
||||
if name.len == 0: return
|
||||
let
|
||||
ps = genParams({"screen_name": name, "trim_user": "true"},
|
||||
count="18", ext=false)
|
||||
url = photoRail ? ps
|
||||
result = parsePhotoRail(await fetch(url, Api.photoRail))
|
||||
proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
|
||||
if id.len == 0: return
|
||||
let js = await fetch(mediaUrl(id, ""))
|
||||
result = parseGraphPhotoRail(js)
|
||||
|
||||
proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
|
||||
let client = newAsyncHttpClient(maxRedirects=0)
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import httpclient, asyncdispatch, options, strutils, uri, times, tables, math
|
||||
import jsony, packedjson, zippy
|
||||
import types, tokens, consts, parserutils, http_pool
|
||||
import types, consts, parserutils, http_pool, tid
|
||||
import experimental/types/common
|
||||
import config
|
||||
|
||||
@ -9,42 +9,34 @@ const
|
||||
rlRemaining = "x-rate-limit-remaining"
|
||||
rlReset = "x-rate-limit-reset"
|
||||
|
||||
var pool: HttpPool
|
||||
var
|
||||
pool: HttpPool
|
||||
disableTid: bool
|
||||
|
||||
proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
|
||||
count="20"; ext=true): seq[(string, string)] =
|
||||
result = timelineParams
|
||||
for p in pars:
|
||||
result &= p
|
||||
if ext:
|
||||
result &= ("include_ext_alt_text", "1")
|
||||
result &= ("include_ext_media_stats", "1")
|
||||
result &= ("include_ext_media_availability", "1")
|
||||
if count.len > 0:
|
||||
result &= ("count", count)
|
||||
if cursor.len > 0:
|
||||
# The raw cursor often has plus signs, which sometimes get turned into spaces,
|
||||
# so we need to turn them back into a plus
|
||||
if " " in cursor:
|
||||
result &= ("cursor", cursor.replace(" ", "+"))
|
||||
else:
|
||||
result &= ("cursor", cursor)
|
||||
proc setDisableTid*(disable: bool) =
|
||||
disableTid = disable
|
||||
|
||||
proc toUrl(req: ApiReq): Uri =
|
||||
let c = req.cookie
|
||||
parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
|
||||
|
||||
proc rateLimitError*(): ref RateLimitError =
|
||||
newException(RateLimitError, "rate limited")
|
||||
|
||||
#proc genHeaders*(token: Token = nil): HttpHeaders =
|
||||
proc genHeaders*(): HttpHeaders =
|
||||
let
|
||||
t = getTime()
|
||||
ffVersion = floor(124 + (t.toUnix() - 1710892800) / 2419200)
|
||||
result = newHttpHeaders({
|
||||
"Connection": "keep-alive",
|
||||
"Authorization": auth,
|
||||
"Authorization": bearerToken,
|
||||
"Content-Type": "application/json",
|
||||
"Accept-Encoding": "gzip",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Accept": "*/*",
|
||||
"DNT": "1",
|
||||
"Host": "x.com",
|
||||
"Origin": "https://x.com/",
|
||||
"Origin": "https://x.com",
|
||||
"Sec-Fetch-Dest": "empty",
|
||||
"Sec-Fetch-Mode": "cors",
|
||||
"Sec-Fetch-Site": "same-origin",
|
||||
@ -56,30 +48,20 @@ proc genHeaders*(): HttpHeaders =
|
||||
"x-twitter-client-language": "en"
|
||||
}, true)
|
||||
|
||||
#template updateToken() =
|
||||
# if resp.headers.hasKey(rlRemaining):
|
||||
# let
|
||||
# remaining = parseInt(resp.headers[rlRemaining])
|
||||
# reset = parseInt(resp.headers[rlReset])
|
||||
# token.setRateLimit(api, remaining, reset)
|
||||
|
||||
template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
|
||||
once:
|
||||
pool = HttpPool()
|
||||
|
||||
#var token = await getToken(api)
|
||||
#if token.tok.len == 0:
|
||||
# raise rateLimitError()
|
||||
|
||||
if len(cfg.cookieHeader) != 0:
|
||||
additional_headers.add("Cookie", cfg.cookieHeader)
|
||||
if not disableTid:
|
||||
additional_headers.add("x-client-transaction-id", await genTid(url.path))
|
||||
|
||||
if len(cfg.xCsrfToken) != 0:
|
||||
additional_headers.add("x-csrf-token", cfg.xCsrfToken)
|
||||
|
||||
try:
|
||||
var resp: AsyncResponse
|
||||
#var headers = genHeaders(token)
|
||||
var headers = genHeaders()
|
||||
for key, value in additional_headers.pairs():
|
||||
headers.add(key, value)
|
||||
@ -102,7 +84,6 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
|
||||
let
|
||||
remaining = parseInt(resp.headers[rlRemaining])
|
||||
reset = parseInt(resp.headers[rlReset])
|
||||
#token.setRateLimit(api, remaining, reset)
|
||||
|
||||
if result.len > 0:
|
||||
if resp.headers.getOrDefault("content-encoding") == "gzip":
|
||||
@ -111,39 +92,27 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} =
|
||||
if result.startsWith("{\"errors"):
|
||||
let errors = result.fromJson(Errors)
|
||||
if errors in {expiredToken, badToken, authorizationError}:
|
||||
echo "fetch error: ", errors
|
||||
#release(token, invalid=true)
|
||||
raise rateLimitError()
|
||||
elif errors in {rateLimited}:
|
||||
# rate limit hit, resets after 24 hours
|
||||
#setLimited(account, api)
|
||||
raise rateLimitError()
|
||||
elif result.startsWith("429 Too Many Requests"):
|
||||
#echo "[accounts] 429 error, API: ", api, ", token: ", token[]
|
||||
#account.apis[api].remaining = 0
|
||||
# rate limit hit, resets after the 15 minute window
|
||||
raise rateLimitError()
|
||||
|
||||
fetchBody
|
||||
|
||||
#release(token, used=true)
|
||||
|
||||
if resp.status == $Http400:
|
||||
let errText = "body: '" & result & "' url: " & $url
|
||||
raise newException(InternalError, errText)
|
||||
echo "ERROR 400, ", url.path, ": ", result
|
||||
raise newException(InternalError, $url)
|
||||
except InternalError as e:
|
||||
raise e
|
||||
except BadClientError as e:
|
||||
#release(token, used=true)
|
||||
raise e
|
||||
except OSError as e:
|
||||
raise e
|
||||
except ProtocolError as e:
|
||||
raise e
|
||||
except Exception as e:
|
||||
echo "error: ", e.name, ", msg: ", e.msg, ", url: ", url
|
||||
#if "length" notin e.msg and "descriptor" notin e.msg:
|
||||
#release(token, invalid=true)
|
||||
echo "error: ", e.name, ", msg: ", e.msg, ", url: ", url.path
|
||||
raise rateLimitError()
|
||||
|
||||
template retry(bod) =
|
||||
@ -152,36 +121,26 @@ template retry(bod) =
|
||||
except ProtocolError:
|
||||
bod
|
||||
|
||||
proc fetch*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
|
||||
proc fetch*(req: ApiReq; additional_headers: HttpHeaders = newHttpHeaders()): Future[JsonNode] {.async.} =
|
||||
retry:
|
||||
var body: string
|
||||
let url = req.toUrl()
|
||||
fetchImpl(body, additional_headers):
|
||||
if body.startsWith('{') or body.startsWith('['):
|
||||
result = parseJson(body)
|
||||
else:
|
||||
echo resp.status, ": ", body, " --- url: ", url
|
||||
echo resp.status, " - non-json for: ", url, ", body: ", result
|
||||
result = newJNull()
|
||||
|
||||
#updateToken()
|
||||
|
||||
let error = result.getError
|
||||
if error in {expiredToken, badToken}:
|
||||
echo "fetch error: ", result.getError
|
||||
#release(token, invalid=true)
|
||||
echo "Fetch error, API: ", url.path, ", error: ", result.getError
|
||||
raise rateLimitError()
|
||||
|
||||
proc fetchRaw*(url: Uri; api: Api; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
|
||||
proc fetchRaw*(req: ApiReq; additional_headers: HttpHeaders = newHttpHeaders()): Future[string] {.async.} =
|
||||
retry:
|
||||
let url = req.toUrl()
|
||||
fetchImpl(result, additional_headers):
|
||||
if not (result.startsWith('{') or result.startsWith('[')):
|
||||
echo resp.status, ": ", result, " --- url: ", url
|
||||
echo resp.status, " - non-json for: ", url, ", body: ", result
|
||||
result.setLen(0)
|
||||
|
||||
#updateToken()
|
||||
|
||||
if result.startsWith("{\"errors"):
|
||||
let errors = result.fromJson(Errors)
|
||||
if errors in {expiredToken, badToken}:
|
||||
echo "fetch error: ", errors
|
||||
#release(token, invalid=true)
|
||||
raise rateLimitError()
|
||||
|
||||
@ -126,7 +126,7 @@ proc getAccountPoolDebug*(): JsonNode =
|
||||
proc rateLimitError*(): ref RateLimitError =
|
||||
newException(RateLimitError, "rate limited")
|
||||
|
||||
proc isLimited(account: GuestAccount; api: Api): bool =
|
||||
proc isLimited(account: GuestAccount; req: ApiReq): bool =
|
||||
if account.isNil:
|
||||
return true
|
||||
|
||||
@ -157,9 +157,9 @@ proc release*(account: GuestAccount) =
|
||||
if account.isNil: return
|
||||
dec account.pending
|
||||
|
||||
proc getGuestAccount*(api: Api): Future[GuestAccount] {.async.} =
|
||||
proc getGuestAccount*(req: ApiReq): Future[GuestAccount] {.async.} =
|
||||
for i in 0 ..< accountPool.len:
|
||||
if result.isReady(api): break
|
||||
if result.isReady(req): break
|
||||
result = accountPool.sample()
|
||||
|
||||
if not result.isNil and result.isReady(api):
|
||||
|
||||
@ -42,6 +42,7 @@ proc getConfig*(path: string): (Config, parseCfg.Config) =
|
||||
enableDebug: cfg.get("Config", "enableDebug", false),
|
||||
proxy: cfg.get("Config", "proxy", ""),
|
||||
proxyAuth: cfg.get("Config", "proxyAuth", ""),
|
||||
disableTid: cfg.get("Config", "disableTid", false),
|
||||
cookieHeader: cfg.get("Config", "cookieHeader", ""),
|
||||
xCsrfToken: cfg.get("Config", "xCsrfToken", "")
|
||||
)
|
||||
|
||||
168
src/consts.nim
168
src/consts.nim
@ -1,54 +1,30 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import uri, sequtils, strutils
|
||||
import strutils
|
||||
|
||||
const
|
||||
auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
|
||||
|
||||
consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
|
||||
consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys"
|
||||
bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
|
||||
|
||||
api = parseUri("https://x.com/i/api")
|
||||
activate* = $(api / "1.1/guest/activate.json")
|
||||
|
||||
photoRail* = api / "1.1/statuses/media_timeline.json"
|
||||
|
||||
timelineApi = api / "2/timeline"
|
||||
|
||||
graphql = api / "graphql"
|
||||
graphUser* = graphql / "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
|
||||
graphUserById* = graphql / "Bbaot8ySMtJD7K2t01gW7A/UserByRestId"
|
||||
graphUserTweets* = graphql / "lZRf8IC-GTuGxDwcsHW8aw/UserTweets"
|
||||
graphUserTweetsAndReplies* = graphql / "gXCeOBFsTOuimuCl1qXimg/UserTweetsAndReplies"
|
||||
graphUserMedia* = graphql / "1D04dx9H2pseMQAbMjXTvQ/UserMedia"
|
||||
graphTweet* = graphql / "6QzqakNMdh_YzBAR9SYPkQ/TweetDetail"
|
||||
graphTweetResult* = graphql / "kLXoXTloWpv9d2FSXRg-Tg/TweetResultByRestId"
|
||||
graphTweetHistory* = graphql / "WT7HhrzWulh4yudKJaR10Q/TweetEditHistory"
|
||||
graphSearchTimeline* = graphql / "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
|
||||
graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
|
||||
graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
|
||||
graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
|
||||
graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
|
||||
graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters"
|
||||
graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters"
|
||||
graphFollowers* = graphql / "SCu9fVIlCUm-BM8-tL5pkQ/Followers"
|
||||
graphFollowing* = graphql / "S5xUN9s2v4xk50KWGGvyvQ/Following"
|
||||
favorites* = graphql / "eSSNbhECHHWWALkkQq-YTA/Likes"
|
||||
|
||||
timelineParams* = {
|
||||
"include_can_media_tag": "1",
|
||||
"include_cards": "1",
|
||||
"include_entities": "1",
|
||||
"include_profile_interstitial_type": "0",
|
||||
"include_quote_count": "0",
|
||||
"include_reply_count": "0",
|
||||
"include_user_entities": "0",
|
||||
"include_ext_reply_count": "0",
|
||||
"include_ext_media_color": "0",
|
||||
"cards_platform": "Web-13",
|
||||
"tweet_mode": "extended",
|
||||
"send_error_codes": "1",
|
||||
"simple_quoted_tweet": "1"
|
||||
}.toSeq
|
||||
graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
|
||||
graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
|
||||
graphUserById* = "Bbaot8ySMtJD7K2t01gW7A/UserByRestId"
|
||||
graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
|
||||
graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
|
||||
graphUserTweets* = "lZRf8IC-GTuGxDwcsHW8aw/UserTweets"
|
||||
graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
|
||||
graphUserMedia* = "1D04dx9H2pseMQAbMjXTvQ/UserMedia"
|
||||
graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
|
||||
graphTweet* = "Y4Erk_-0hObvLpz0Iw3bzA/ConversationTimeline"
|
||||
graphTweetDetail* = "6QzqakNMdh_YzBAR9SYPkQ/TweetDetail"
|
||||
graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
|
||||
graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
|
||||
graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
|
||||
graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
|
||||
graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
|
||||
graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
|
||||
graphFollowers* = "Efm7xwLreAw77q2Fq7rX-Q/Followers"
|
||||
graphFollowing* = "e0UtTAwQqgLKBllQxMgVxQ/Following"
|
||||
|
||||
gqlFeatures* = """{
|
||||
"rweb_video_screen_enabled": false,
|
||||
@ -84,85 +60,77 @@ const
|
||||
"responsive_web_grok_image_annotation_enabled": true,
|
||||
"responsive_web_grok_imagine_annotation_enabled": true,
|
||||
"responsive_web_grok_community_note_auto_translation_is_enabled": false,
|
||||
"responsive_web_enhance_cards_enabled": false
|
||||
"responsive_web_enhance_cards_enabled": false,
|
||||
"payments_enabled": false,
|
||||
"responsive_web_twitter_article_notes_tab_enabled": false,
|
||||
"hidden_profile_subscriptions_enabled": false,
|
||||
"subscriptions_verification_info_verified_since_enabled": false,
|
||||
"subscriptions_verification_info_is_identity_verified_enabled": false,
|
||||
"highlights_tweets_tab_ui_enabled": false,
|
||||
"subscriptions_feature_can_gift_premium": false
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
userFeatures* = """{
|
||||
"hidden_profile_subscriptions_enabled": true,
|
||||
"profile_label_improvements_pcf_label_in_post_enabled": true,
|
||||
"responsive_web_profile_redirect_enabled": false,
|
||||
"rweb_tipjar_consumption_enabled": true,
|
||||
"verified_phone_label_enabled": false,
|
||||
"subscriptions_verification_info_is_identity_verified_enabled": true,
|
||||
"subscriptions_verification_info_verified_since_enabled": true,
|
||||
"highlights_tweets_tab_ui_enabled": true,
|
||||
"responsive_web_twitter_article_notes_tab_enabled": true,
|
||||
"subscriptions_feature_can_gift_premium": true,
|
||||
"creator_subscriptions_tweet_preview_api_enabled": true,
|
||||
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
|
||||
"responsive_web_graphql_timeline_navigation_enabled": true
|
||||
tweetVars* = """{
|
||||
"postId": "$1",
|
||||
$2
|
||||
"includeHasBirdwatchNotes": false,
|
||||
"includePromotedContent": false,
|
||||
"withBirdwatchNotes": false,
|
||||
"withVoice": false,
|
||||
"withV2Timeline": true
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
tweetVariables* = """{
|
||||
tweetDetailVars* = """{
|
||||
"focalTweetId": "$1",
|
||||
$2
|
||||
"referrer": "profile",
|
||||
"with_rux_injections": false,
|
||||
"rankingMode": "Relevance",
|
||||
"includePromotedContent": false,
|
||||
"includePromotedContent": true,
|
||||
"withCommunity": true,
|
||||
"withQuickPromoteEligibilityTweetFields": false,
|
||||
"withQuickPromoteEligibilityTweetFields": true,
|
||||
"withBirdwatchNotes": true,
|
||||
"withVoice": true
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
tweetFieldToggles* = """{
|
||||
"withArticleRichContentState": true,
|
||||
"withArticlePlainText": false,
|
||||
"withGrokAnalyze": false,
|
||||
"withDisallowedReplyControls": false
|
||||
restIdVars* = """{
|
||||
"rest_id": "$1", $2
|
||||
"count": 20
|
||||
}"""
|
||||
|
||||
userMediaVars* = """{
|
||||
"userId": "$1", $2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withClientEventToken": false,
|
||||
"withBirdwatchNotes": false,
|
||||
"withVoice": true
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
userTweetsVariables* = """{
|
||||
"userId": "$1",
|
||||
$2
|
||||
userTweetsVars* = """{
|
||||
"userId": "$1", $2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withQuickPromoteEligibilityTweetFields": true,
|
||||
"withVoice": true
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
userTweetsAndRepliesVars* = """{
|
||||
"userId": "$1", $2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withCommunity": true,
|
||||
"withVoice": true
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
listTweetsVariables* = """{
|
||||
"rest_id": "$1",
|
||||
$2
|
||||
"count": 20
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
reactorsVariables* = """{
|
||||
"tweetId": "$1",
|
||||
followVars* = """{
|
||||
"userId": "$1",
|
||||
$2
|
||||
"count": 20,
|
||||
"includePromotedContent": false
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
followVariables* = """{
|
||||
"userId": "$1",
|
||||
$2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withGrokTranslatedBio": false
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
userMediaVariables* = """{
|
||||
"userId": "$1",
|
||||
$2
|
||||
"count": 20,
|
||||
"includePromotedContent": false,
|
||||
"withClientEventToken": false,
|
||||
"withBirdwatchNotes": true,
|
||||
"withVoice": true
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
|
||||
tweetHistoryVariables* = """{
|
||||
"tweetId": "$1",
|
||||
"withQuickPromoteEligibilityTweetFields": false
|
||||
}""".replace(" ", "").replace("\n", "")
|
||||
userFieldToggles = """{"withPayments":false,"withAuxiliaryUserLabels":true}"""
|
||||
userTweetsFieldToggles* = """{"withArticlePlainText":false}"""
|
||||
tweetDetailFieldToggles* = """{"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false}"""
|
||||
|
||||
@ -1,21 +1,53 @@
|
||||
import options
|
||||
import options, strutils
|
||||
import jsony
|
||||
import user, ../types/[graphuser, graphlistmembers]
|
||||
import user, utils, ../types/[graphuser, graphlistmembers]
|
||||
from ../../types import User, VerifiedType, Result, Query, QueryKind
|
||||
|
||||
#proc parseGraphUser*(json: string): User =
|
||||
# if json.len == 0 or json[0] != '{':
|
||||
# return
|
||||
#
|
||||
# let raw = json.fromJson(GraphUser)
|
||||
#
|
||||
# if raw.data.userResult.result.unavailableReason.get("") == "Suspended":
|
||||
# return User(suspended: true)
|
||||
#
|
||||
# result = raw.data.userResult.result.legacy
|
||||
# result.id = raw.data.userResult.result.restId
|
||||
# if result.verifiedType == VerifiedType.none and raw.data.userResult.result.isBlueVerified:
|
||||
# result.verifiedType = blue
|
||||
proc parseUserResult*(userResult: UserResult): User =
|
||||
result = userResult.legacy
|
||||
|
||||
if result.verifiedType == none and userResult.isBlueVerified:
|
||||
result.verifiedType = blue
|
||||
|
||||
if result.username.len == 0 and userResult.core.screenName.len > 0:
|
||||
result.id = userResult.restId
|
||||
result.username = userResult.core.screenName
|
||||
result.fullname = userResult.core.name
|
||||
result.userPic = userResult.avatar.imageUrl.replace("_normal", "")
|
||||
|
||||
if userResult.privacy.isSome:
|
||||
result.protected = userResult.privacy.get.protected
|
||||
|
||||
if userResult.location.isSome:
|
||||
result.location = userResult.location.get.location
|
||||
|
||||
if userResult.core.createdAt.len > 0:
|
||||
result.joinDate = parseTwitterDate(userResult.core.createdAt)
|
||||
|
||||
if userResult.verification.isSome:
|
||||
let v = userResult.verification.get
|
||||
if v.verifiedType != VerifiedType.none:
|
||||
result.verifiedType = v.verifiedType
|
||||
|
||||
if userResult.profileBio.isSome and result.bio.len == 0:
|
||||
result.bio = userResult.profileBio.get.description
|
||||
|
||||
proc parseGraphUser*(json: string): User =
|
||||
if json.len == 0 or json[0] != '{':
|
||||
return
|
||||
|
||||
let
|
||||
raw = json.fromJson(GraphUser)
|
||||
userResult =
|
||||
if raw.data.userResult.isSome: raw.data.userResult.get.result
|
||||
elif raw.data.user.isSome: raw.data.user.get.result
|
||||
else: UserResult()
|
||||
|
||||
if userResult.unavailableReason.get("") == "Suspended" or
|
||||
userResult.reason.get("") == "Suspended":
|
||||
return User(suspended: true)
|
||||
|
||||
result = parseUserResult(userResult)
|
||||
|
||||
proc parseGraphListMembers*(json, cursor: string): Result[User] =
|
||||
result = Result[User](
|
||||
@ -31,7 +63,7 @@ proc parseGraphListMembers*(json, cursor: string): Result[User] =
|
||||
of TimelineTimelineItem:
|
||||
let userResult = entry.content.itemContent.userResults.result
|
||||
if userResult.restId.len > 0:
|
||||
result.content.add userResult.legacy
|
||||
result.content.add parseUserResult(userResult)
|
||||
of TimelineTimelineCursor:
|
||||
if entry.content.cursorType == "Bottom":
|
||||
result.bottom = entry.content.value
|
||||
|
||||
8
src/experimental/parser/tid.nim
Normal file
8
src/experimental/parser/tid.nim
Normal file
@ -0,0 +1,8 @@
|
||||
import jsony
|
||||
import ../types/tid
|
||||
export TidPair
|
||||
|
||||
proc parseTidPairs*(raw: string): seq[TidPair] =
|
||||
result = raw.fromJson(seq[TidPair])
|
||||
if result.len == 0:
|
||||
raise newException(ValueError, "Parsing pairs failed: " & raw)
|
||||
@ -1,15 +1,48 @@
|
||||
import options
|
||||
from ../../types import User
|
||||
import options, strutils
|
||||
from ../../types import User, VerifiedType
|
||||
|
||||
type
|
||||
GraphUser* = object
|
||||
data*: tuple[userResult: UserData]
|
||||
data*: tuple[userResult: Option[UserData], user: Option[UserData]]
|
||||
|
||||
UserData* = object
|
||||
result*: UserResult
|
||||
|
||||
UserResult = object
|
||||
UserCore* = object
|
||||
name*: string
|
||||
screenName*: string
|
||||
createdAt*: string
|
||||
|
||||
UserBio* = object
|
||||
description*: string
|
||||
|
||||
UserAvatar* = object
|
||||
imageUrl*: string
|
||||
|
||||
Verification* = object
|
||||
verifiedType*: VerifiedType
|
||||
|
||||
Location* = object
|
||||
location*: string
|
||||
|
||||
Privacy* = object
|
||||
protected*: bool
|
||||
|
||||
UserResult* = object
|
||||
legacy*: User
|
||||
restId*: string
|
||||
isBlueVerified*: bool
|
||||
core*: UserCore
|
||||
avatar*: UserAvatar
|
||||
unavailableReason*: Option[string]
|
||||
reason*: Option[string]
|
||||
privacy*: Option[Privacy]
|
||||
profileBio*: Option[UserBio]
|
||||
verification*: Option[Verification]
|
||||
location*: Option[Location]
|
||||
|
||||
proc enumHook*(s: string; v: var VerifiedType) =
|
||||
v = try:
|
||||
parseEnum[VerifiedType](s)
|
||||
except:
|
||||
VerifiedType.none
|
||||
|
||||
4
src/experimental/types/tid.nim
Normal file
4
src/experimental/types/tid.nim
Normal file
@ -0,0 +1,4 @@
|
||||
type
|
||||
TidPair* = object
|
||||
animationKey*: string
|
||||
verification*: string
|
||||
@ -6,7 +6,7 @@ from htmlgen import a
|
||||
|
||||
import jester
|
||||
|
||||
import types, config, prefs, formatters, redis_cache, http_pool
|
||||
import types, config, prefs, formatters, redis_cache, http_pool, apiutils
|
||||
import views/[general, about]
|
||||
import routes/[
|
||||
preferences, timeline, status, media, search, list, rss, #debug,
|
||||
@ -34,6 +34,7 @@ setHmacKey(cfg.hmacKey)
|
||||
setProxyEncoding(cfg.base64Media)
|
||||
setMaxHttpConns(cfg.httpMaxConns)
|
||||
setHttpProxy(cfg.proxy, cfg.proxyAuth)
|
||||
setDisableTid(cfg.disableTid)
|
||||
initAboutPage(cfg.staticDir)
|
||||
|
||||
waitFor initRedisPool(cfg)
|
||||
@ -104,7 +105,6 @@ routes:
|
||||
extend preferences, ""
|
||||
extend resolver, ""
|
||||
extend embed, ""
|
||||
#extend debug, ""
|
||||
extend activityspoof, ""
|
||||
extend api, ""
|
||||
extend unsupported, ""
|
||||
|
||||
@ -45,6 +45,7 @@ proc parseGraphUser*(js: JsonNode): User =
|
||||
result.fullname = user{"core", "name"}.getStr
|
||||
result.joinDate = parseTwitterDate(user{"core", "created_at"}.getStr)
|
||||
result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
|
||||
result.protected = user{"privacy", "protected"}.getBool
|
||||
|
||||
let label = user{"affiliates_highlighted_label", "label"}
|
||||
if not label.isNull:
|
||||
@ -449,23 +450,6 @@ proc parseTimeline*(js: JsonNode; after=""): Profile =
|
||||
else:
|
||||
result.tweets.top = cursor{"value"}.getStr
|
||||
|
||||
proc parsePhotoRail*(js: JsonNode): PhotoRail =
|
||||
with error, js{"error"}:
|
||||
if error.getStr == "Not authorized.":
|
||||
return
|
||||
|
||||
for tweet in js:
|
||||
let
|
||||
t = parseTweet(tweet, js{"tweet_card"})
|
||||
url = if t.photos.len > 0: t.photos[0].url
|
||||
elif t.video.isSome: get(t.video).thumb
|
||||
elif t.gif.isSome: get(t.gif).thumb
|
||||
elif t.card.isSome: get(t.card).image
|
||||
else: ""
|
||||
|
||||
if url.len == 0: continue
|
||||
result.add GalleryPhoto(url: url, tweetId: $t.id)
|
||||
|
||||
proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
|
||||
if js.kind == JNull:
|
||||
return Tweet()
|
||||
@ -531,6 +515,34 @@ proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
|
||||
else:
|
||||
result.stats.views = -1
|
||||
|
||||
proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
|
||||
result = @[]
|
||||
|
||||
let instructions =
|
||||
? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
|
||||
|
||||
for i in instructions:
|
||||
if i{"__typename"}.getStr == "TimelineAddEntries":
|
||||
for e in i{"entries"}:
|
||||
let entryId = e{"entryId"}.getStr
|
||||
if entryId.startsWith("tweet"):
|
||||
with tweetResult, e{"content", "content", "tweetResult", "result"}:
|
||||
let t = parseGraphTweet(tweetResult, false)
|
||||
if not t.available:
|
||||
t.id = $parseBiggestInt(entryId.getId())
|
||||
|
||||
let url =
|
||||
if t.photos.len > 0: t.photos[0].url
|
||||
elif t.video.isSome: get(t.video).thumb
|
||||
elif t.gif.isSome: get(t.gif).thumb
|
||||
elif t.card.isSome: get(t.card).image
|
||||
else: ""
|
||||
|
||||
result.add GalleryPhoto(url: url, tweetId: t.id)
|
||||
|
||||
if result.len == 16:
|
||||
break
|
||||
|
||||
proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
|
||||
for t in js{"content", "items"}:
|
||||
let entryId = t{"entryId"}.getStr
|
||||
|
||||
@ -87,7 +87,7 @@ proc cache*(data: List) {.async.} =
|
||||
await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
|
||||
|
||||
proc cache*(data: PhotoRail; name: string) {.async.} =
|
||||
await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
|
||||
await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
|
||||
|
||||
proc cache*(data: User) {.async.} =
|
||||
if data.username.len == 0: return
|
||||
@ -145,6 +145,9 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
|
||||
elif fetch:
|
||||
result = await getGraphUser(username)
|
||||
await cache(result)
|
||||
if result.id.len > 0:
|
||||
await setEx("i:" & result.id, baseCacheTime, result.username)
|
||||
await cacheUserId(result.username, result.id)
|
||||
|
||||
proc getCachedUsername*(userId: string): Future[string] {.async.} =
|
||||
let
|
||||
@ -174,14 +177,14 @@ proc getCachedTweet*(id: string; after=""): Future[Conversation] {.async.} =
|
||||
if not result.isNil and after.len > 0:
|
||||
result.replies = await getReplies(id, after)
|
||||
|
||||
proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
|
||||
if name.len == 0: return
|
||||
let rail = await get("pr:" & toLower(name))
|
||||
proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
|
||||
if id.len == 0: return
|
||||
let rail = await get("pr2:" & toLower(id))
|
||||
if rail != redisNil:
|
||||
rail.deserialize(PhotoRail)
|
||||
else:
|
||||
result = await getPhotoRail(name)
|
||||
await cache(result, name)
|
||||
result = await getPhotoRail(id)
|
||||
await cache(result, id)
|
||||
|
||||
proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
|
||||
let list = if id.len == 0: redisNil
|
||||
|
||||
@ -19,14 +19,15 @@ proc createActivityPubRouter*(cfg: Config) =
|
||||
|
||||
get "/api/v1/accounts/@id":
|
||||
let id = @"id"
|
||||
if id.len > 19 or id.any(c => not c.isDigit):
|
||||
#if id.len > 19 or id.any(c => not c.isDigit):
|
||||
if not id.allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_'}):
|
||||
resp Http404, {"Content-Type": "application/json"}, """{"error":"Invalid account ID"}"""
|
||||
|
||||
var username = await getCachedUsername(id)
|
||||
if username.len == 0:
|
||||
resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
|
||||
#var username = await getCachedUsername(id)
|
||||
#if username.len == 0:
|
||||
#resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
|
||||
|
||||
let user = await getCachedUser(username)
|
||||
let user = await getCachedUser(id)
|
||||
if user.suspended or user.id.len == 0:
|
||||
resp Http404, {"Content-Type": "application/json"}, """{"error":"User not found"}"""
|
||||
|
||||
@ -189,9 +190,8 @@ proc createActivityPubRouter*(cfg: Config) =
|
||||
postJson["edited_at"] = newJNull()
|
||||
postJson["reblog"] = newJNull()
|
||||
if tweet.replyId.len != 0:
|
||||
let replyUser = await getCachedUser(tweet.replyHandle)
|
||||
postJson["in_reply_to_id"] = %(&"{tweet.replyId}")
|
||||
postJson["in_reply_to_account_id"] = %replyUser.id
|
||||
postJson["in_reply_to_account_id"] = %tweet.replyHandle
|
||||
else:
|
||||
postJson["in_reply_to_id"] = newJNull()
|
||||
postJson["in_reply_to_account_id"] = newJNull()
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import jester
|
||||
import router_utils
|
||||
import ".."/[tokens, types]
|
||||
|
||||
proc createDebugRouter*(cfg: Config) =
|
||||
router debug:
|
||||
get "/.tokens":
|
||||
cond cfg.enableDebug
|
||||
respJson getPoolJson()
|
||||
|
||||
#get "/.health":
|
||||
#respJson getAccountPoolHealth()
|
||||
|
||||
#get "/.accounts":
|
||||
#cond cfg.enableDebug
|
||||
#respJson getAccountPoolDebug()
|
||||
@ -5,7 +5,7 @@ import jester, karax/vdom
|
||||
|
||||
import router_utils
|
||||
import ".."/[types, formatters, api, redis_cache]
|
||||
import ../views/[general, status, search, mastoapi]
|
||||
import ../views/[general, status, mastoapi]
|
||||
|
||||
export json, uri, sequtils, options, sugar, times
|
||||
export router_utils
|
||||
|
||||
@ -48,7 +48,7 @@ proc fetchProfile*(after: string; query: Query; cfg: Config; skipRail=false;
|
||||
let
|
||||
rail =
|
||||
skipIf(skipRail or query.kind == media, @[]):
|
||||
getCachedPhotoRail(name)
|
||||
getCachedPhotoRail(userId)
|
||||
|
||||
user = getCachedUser(name)
|
||||
|
||||
@ -111,6 +111,7 @@ proc createTimelineRouter*(cfg: Config) =
|
||||
get "/@name/?@tab?/?":
|
||||
cond '.' notin @"name"
|
||||
cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
|
||||
cond @"name".allCharsInSet({'a'..'z', 'A'..'Z', '0'..'9', '_', ','})
|
||||
cond @"tab" in ["with_replies", "media", "search", "following", "followers", ""]
|
||||
let
|
||||
prefs = cookiePrefs()
|
||||
|
||||
@ -4,7 +4,7 @@ import json, asyncdispatch, options, uri
|
||||
import times
|
||||
import jester
|
||||
import router_utils
|
||||
import ".."/[types, api, apiutils, query, consts, redis_cache]
|
||||
import ".."/[types, api, apiutils, redis_cache]
|
||||
import httpclient, strutils
|
||||
import sequtils
|
||||
|
||||
@ -81,60 +81,49 @@ proc getUserTweetsJson*(id: string): Future[JsonNode] {.async.} =
|
||||
|
||||
result = response
|
||||
|
||||
proc searchTimeline*(query: Query; after=""): Future[string] {.async.} =
|
||||
let q = genQueryParam(query)
|
||||
var
|
||||
variables = %*{
|
||||
"rawQuery": q,
|
||||
"count": 20,
|
||||
"product": "Latest",
|
||||
"withDownvotePerspective": false,
|
||||
"withReactionsMetadata": false,
|
||||
"withReactionsPerspective": false
|
||||
}
|
||||
if after.len > 0:
|
||||
variables["cursor"] = % after
|
||||
let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
|
||||
result = await fetchRaw(url, Api.search)
|
||||
|
||||
proc getUserTweets*(id: string; after=""): Future[string] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = userTweetsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
result = await fetchRaw(graphUserTweets ? params, Api.userTweets)
|
||||
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/$1""" % id)
|
||||
|
||||
let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
result = await fetchRaw(userTweetsUrl(id, cursor), headers)
|
||||
|
||||
proc getUserReplies*(id: string; after=""): Future[string] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = userTweetsVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
result = await fetchRaw(graphUserTweets ? params, Api.userTweetsAndReplies)
|
||||
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/$1/with_replies""" % id)
|
||||
|
||||
let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
result = await fetchRaw(userTweetsAndRepliesUrl(id, cursor), headers)
|
||||
|
||||
proc getUserMedia*(id: string; after=""): Future[string] {.async.} =
|
||||
if id.len == 0: return
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = userMediaVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
result = await fetchRaw(graphUserMedia ? params, Api.userMedia)
|
||||
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/$1/media""" % id)
|
||||
|
||||
let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
result = await fetchRaw(mediaUrl(id, cursor), headers)
|
||||
|
||||
proc getTweetById*(id: string; after=""): Future[string] {.async.} =
|
||||
let
|
||||
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
variables = tweetVariables % [id, cursor]
|
||||
params = {"variables": variables, "features": gqlFeatures}
|
||||
result = await fetchRaw(graphTweet ? params, Api.tweetDetail)
|
||||
if id.len == 0: return
|
||||
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/i/status/$1""" % id)
|
||||
|
||||
let cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
|
||||
result = await fetchRaw(tweetDetailUrl(id, cursor), headers)
|
||||
|
||||
proc getUser*(username: string): Future[string] {.async.} =
|
||||
if username.len == 0: return
|
||||
let
|
||||
variables = """{"screen_name":"$1"}""" % username
|
||||
fieldToggles = """{"withAuxiliaryUserLabels":true}"""
|
||||
params = {"variables": variables, "features": gqlFeatures, "fieldToggles": fieldToggles}
|
||||
result = await fetchRaw(graphUser ? params, Api.userScreenName)
|
||||
|
||||
let headers = newHttpHeaders()
|
||||
headers.add("Referer", """https://x.com/$1""" % username)
|
||||
|
||||
result = await fetchRaw(userUrl(username), headers)
|
||||
|
||||
proc createTwitterApiRouter*(cfg: Config) =
|
||||
router api:
|
||||
@ -146,16 +135,11 @@ proc createTwitterApiRouter*(cfg: Config) =
|
||||
let response = await getUser(username)
|
||||
resp Http200, { "Content-Type": "application/json" }, response
|
||||
|
||||
#get "/api/user/@id/tweets":
|
||||
# let id = @"id"
|
||||
# let response = await getUserTweetsJson(id)
|
||||
# respJson response
|
||||
|
||||
get "/api/user/@username/timeline":
|
||||
let username = @"username"
|
||||
let query = Query(fromUser: @[username])
|
||||
let response = await searchTimeline(query)
|
||||
resp Http200, { "Content-Type": "application/json" }, response
|
||||
#get "/api/user/@username/timeline":
|
||||
# let username = @"username"
|
||||
# let query = Query(fromUser: @[username])
|
||||
# let response = await searchTimeline(query)
|
||||
# resp Http200, { "Content-Type": "application/json" }, response
|
||||
|
||||
get "/api/user/@id/tweets":
|
||||
let id = @"id"
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
font-weight: bold;
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
padding: 0px 5px 1px 8px;
|
||||
}
|
||||
|
||||
input {
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
@import '_variables';
|
||||
@import "_variables";
|
||||
|
||||
@import 'tweet/_base';
|
||||
@import 'profile/_base';
|
||||
@import 'general';
|
||||
@import 'navbar';
|
||||
@import 'inputs';
|
||||
@import 'timeline';
|
||||
@import 'search';
|
||||
@import "tweet/_base";
|
||||
@import "profile/_base";
|
||||
@import "general";
|
||||
@import "navbar";
|
||||
@import "inputs";
|
||||
@import "timeline";
|
||||
@import "search";
|
||||
|
||||
body {
|
||||
// colors
|
||||
@ -50,7 +50,7 @@ body {
|
||||
|
||||
background-color: var(--bg_color);
|
||||
color: var(--fg_color);
|
||||
font-family: $font_0, $font_1, $font_2, $font_3;
|
||||
font-family: $font_0, $font_1;
|
||||
font-size: 14px;
|
||||
line-height: 1.3;
|
||||
margin: 0;
|
||||
@ -66,7 +66,8 @@ h1 {
|
||||
display: inline;
|
||||
}
|
||||
|
||||
h2, h3 {
|
||||
h2,
|
||||
h3 {
|
||||
font-weight: normal;
|
||||
}
|
||||
|
||||
@ -90,7 +91,7 @@ fieldset {
|
||||
|
||||
legend {
|
||||
width: 100%;
|
||||
padding: .6em 0 .3em 0;
|
||||
padding: 0.6em 0 0.3em 0;
|
||||
border: 0;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
@ -142,30 +143,50 @@ ul {
|
||||
}
|
||||
|
||||
.verified-icon {
|
||||
color: var(--icon_text);
|
||||
border-radius: 50%;
|
||||
flex-shrink: 0;
|
||||
margin: 2px 0 3px 3px;
|
||||
padding-top: 3px;
|
||||
height: 11px;
|
||||
width: 14px;
|
||||
font-size: 8px;
|
||||
display: inline-block;
|
||||
text-align: center;
|
||||
vertical-align: middle;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
margin-left: 2px;
|
||||
|
||||
.verified-icon-circle {
|
||||
position: absolute;
|
||||
font-size: 15px;
|
||||
}
|
||||
|
||||
.verified-icon-check {
|
||||
position: absolute;
|
||||
font-size: 9px;
|
||||
margin: 5px 3px;
|
||||
}
|
||||
|
||||
&.blue {
|
||||
background-color: var(--verified_blue);
|
||||
.verified-icon-circle {
|
||||
color: var(--verified_blue);
|
||||
}
|
||||
|
||||
.verified-icon-check {
|
||||
color: var(--icon_text);
|
||||
}
|
||||
}
|
||||
|
||||
&.business {
|
||||
.verified-icon-circle {
|
||||
color: var(--verified_business);
|
||||
}
|
||||
|
||||
.verified-icon-check {
|
||||
color: var(--bg_panel);
|
||||
background-color: var(--verified_business);
|
||||
}
|
||||
}
|
||||
|
||||
&.government {
|
||||
.verified-icon-circle {
|
||||
color: var(--verified_government);
|
||||
}
|
||||
|
||||
.verified-icon-check {
|
||||
color: var(--bg_panel);
|
||||
background-color: var(--verified_government);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -174,7 +195,8 @@ ul {
|
||||
max-width: 95vw;
|
||||
}
|
||||
|
||||
.nav-item, .nav-item .icon-container {
|
||||
.nav-item,
|
||||
.nav-item .icon-container {
|
||||
font-size: 16px;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
@import '_variables';
|
||||
@import "_variables";
|
||||
|
||||
nav {
|
||||
display: flex;
|
||||
@ -12,7 +12,8 @@ nav {
|
||||
z-index: 1000;
|
||||
font-size: 16px;
|
||||
|
||||
a, .icon-button button {
|
||||
a,
|
||||
.icon-button button {
|
||||
color: var(--fg_nav);
|
||||
}
|
||||
}
|
||||
@ -58,15 +59,11 @@ nav {
|
||||
justify-content: flex-end;
|
||||
}
|
||||
|
||||
&.right a {
|
||||
padding-left: 4px;
|
||||
|
||||
&:hover {
|
||||
&.right a:hover {
|
||||
color: var(--accent_light);
|
||||
text-decoration: unset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.lp {
|
||||
height: 14px;
|
||||
@ -80,10 +77,11 @@ nav {
|
||||
}
|
||||
}
|
||||
|
||||
.icon-info:before {
|
||||
.icon-info {
|
||||
margin: 0 -3px;
|
||||
}
|
||||
|
||||
.icon-cog {
|
||||
font-size: 15px;
|
||||
padding-left: 0 !important;
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
@import '_variables';
|
||||
@import '_mixins';
|
||||
@import "_variables";
|
||||
@import "_mixins";
|
||||
|
||||
.search-title {
|
||||
font-weight: bold;
|
||||
@ -13,6 +13,7 @@
|
||||
|
||||
button {
|
||||
margin: 0 2px 0 0;
|
||||
padding: 0px 1px 1px 4px;
|
||||
height: 23px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@ -34,7 +35,7 @@
|
||||
background-color: var(--bg_elements);
|
||||
color: var(--fg_color);
|
||||
border: 1px solid var(--accent_border);
|
||||
padding: 1px 6px 2px 6px;
|
||||
padding: 1px 1px 2px 4px;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
margin-bottom: 2px;
|
||||
@ -62,7 +63,7 @@
|
||||
.checkbox-container {
|
||||
display: inline;
|
||||
padding-right: unset;
|
||||
margin-bottom: unset;
|
||||
margin-bottom: 5px;
|
||||
margin-left: 23px;
|
||||
}
|
||||
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
@import '_variables';
|
||||
@import '_mixins';
|
||||
@import "_variables";
|
||||
@import "_mixins";
|
||||
|
||||
video {
|
||||
max-height: 100%;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
@ -13,14 +13,12 @@ video {
|
||||
|
||||
.gallery-video.card-container {
|
||||
flex-direction: column;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.video-container {
|
||||
max-height: 530px;
|
||||
margin: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
|
||||
img {
|
||||
max-height: 100%;
|
||||
|
||||
62
src/tid.nim
Normal file
62
src/tid.nim
Normal file
@ -0,0 +1,62 @@
|
||||
import std/[asyncdispatch, base64, httpclient, random, strutils, sequtils, times]
|
||||
import nimcrypto
|
||||
import experimental/parser/tid
|
||||
|
||||
randomize()
|
||||
|
||||
const defaultKeyword = "obfiowerehiring";
|
||||
const pairsUrl =
|
||||
"https://raw.githubusercontent.com/fa0311/x-client-transaction-id-pair-dict/refs/heads/main/pair.json";
|
||||
|
||||
var
|
||||
cachedPairs: seq[TidPair] = @[]
|
||||
lastCached = 0
|
||||
# refresh every hour
|
||||
ttlSec = 60 * 60
|
||||
|
||||
proc getPair(): Future[TidPair] {.async.} =
|
||||
if cachedPairs.len == 0 or int(epochTime()) - lastCached > ttlSec:
|
||||
lastCached = int(epochTime())
|
||||
|
||||
let client = newAsyncHttpClient()
|
||||
defer: client.close()
|
||||
|
||||
let resp = await client.get(pairsUrl)
|
||||
if resp.status == $Http200:
|
||||
cachedPairs = parseTidPairs(await resp.body)
|
||||
|
||||
return sample(cachedPairs)
|
||||
|
||||
proc encodeSha256(text: string): array[32, byte] =
|
||||
let
|
||||
data = cast[ptr byte](addr text[0])
|
||||
dataLen = uint(len(text))
|
||||
digest = sha256.digest(data, dataLen)
|
||||
return digest.data
|
||||
|
||||
proc encodeBase64[T](data: T): string =
|
||||
return encode(data).replace("=", "")
|
||||
|
||||
proc decodeBase64(data: string): seq[byte] =
|
||||
return cast[seq[byte]](decode(data))
|
||||
|
||||
proc genTid*(path: string): Future[string] {.async.} =
|
||||
let
|
||||
pair = await getPair()
|
||||
|
||||
timeNow = int(epochTime() - 1682924400)
|
||||
timeNowBytes = @[
|
||||
byte(timeNow and 0xff),
|
||||
byte((timeNow shr 8) and 0xff),
|
||||
byte((timeNow shr 16) and 0xff),
|
||||
byte((timeNow shr 24) and 0xff)
|
||||
]
|
||||
|
||||
data = "GET!" & path & "!" & $timeNow & defaultKeyword & pair.animationKey
|
||||
hashBytes = encodeSha256(data)
|
||||
keyBytes = decodeBase64(pair.verification)
|
||||
bytesArr = keyBytes & timeNowBytes & hashBytes[0 ..< 16] & @[3'u8]
|
||||
randomNum = byte(rand(256))
|
||||
tid = @[randomNum] & bytesArr.mapIt(it xor randomNum)
|
||||
|
||||
return encodeBase64(tid)
|
||||
168
src/tokens.nim
168
src/tokens.nim
@ -1,168 +0,0 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
import asyncdispatch, httpclient, times, sequtils, json, random
|
||||
import strutils, tables
|
||||
import types, consts
|
||||
|
||||
const
|
||||
maxConcurrentReqs = 5 # max requests at a time per token, to avoid race conditions
|
||||
maxLastUse = 1.hours # if a token is unused for 60 minutes, it expires
|
||||
maxAge = 2.hours + 55.minutes # tokens expire after 3 hours
|
||||
failDelay = initDuration(minutes=30)
|
||||
|
||||
var
|
||||
tokenPool: seq[Token]
|
||||
lastFailed: Time
|
||||
enableLogging = false
|
||||
|
||||
let headers = newHttpHeaders({"authorization": auth})
|
||||
|
||||
template log(str) =
|
||||
if enableLogging: echo "[tokens] ", str
|
||||
|
||||
proc getPoolJson*(): JsonNode =
|
||||
var
|
||||
list = newJObject()
|
||||
totalReqs = 0
|
||||
totalPending = 0
|
||||
reqsPerApi: Table[string, int]
|
||||
|
||||
for token in tokenPool:
|
||||
totalPending.inc(token.pending)
|
||||
list[token.tok] = %*{
|
||||
"apis": newJObject(),
|
||||
"pending": token.pending,
|
||||
"init": $token.init,
|
||||
"lastUse": $token.lastUse
|
||||
}
|
||||
|
||||
for api in token.apis.keys:
|
||||
list[token.tok]["apis"][$api] = %token.apis[api]
|
||||
|
||||
let
|
||||
maxReqs =
|
||||
case api
|
||||
of Api.photoRail: 180
|
||||
#of Api.timeline: 187
|
||||
#of Api.userTweets, Api.userTimeline: 300
|
||||
of Api.userTweets: 300
|
||||
of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
|
||||
Api.userTweetsAndReplies, Api.userMedia,
|
||||
Api.userRestId, Api.userScreenName, Api.tweetDetail,
|
||||
Api.tweetResult, Api.search, Api.favorites,
|
||||
Api.retweeters, Api.favoriters, Api.following, Api.followers: 500
|
||||
#of Api.userSearch: 900
|
||||
reqs = maxReqs - token.apis[api].remaining
|
||||
|
||||
reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
|
||||
totalReqs.inc(reqs)
|
||||
|
||||
return %*{
|
||||
"amount": tokenPool.len,
|
||||
"requests": totalReqs,
|
||||
"pending": totalPending,
|
||||
"apis": reqsPerApi,
|
||||
"tokens": list
|
||||
}
|
||||
|
||||
proc rateLimitError*(): ref RateLimitError =
|
||||
newException(RateLimitError, "rate limited")
|
||||
|
||||
proc fetchToken(): Future[Token] {.async.} =
|
||||
if getTime() - lastFailed < failDelay:
|
||||
raise rateLimitError()
|
||||
|
||||
let client = newAsyncHttpClient(headers=headers)
|
||||
|
||||
try:
|
||||
let
|
||||
resp = await client.postContent(activate)
|
||||
tokNode = parseJson(resp)["guest_token"]
|
||||
tok = tokNode.getStr($(tokNode.getInt))
|
||||
time = getTime()
|
||||
|
||||
return Token(tok: tok, init: time, lastUse: time)
|
||||
except Exception as e:
|
||||
echo "[tokens] fetching token failed: ", e.msg
|
||||
if "Try again" notin e.msg:
|
||||
echo "[tokens] fetching tokens paused, resuming in 30 minutes"
|
||||
lastFailed = getTime()
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
proc expired(token: Token): bool =
|
||||
let time = getTime()
|
||||
token.init < time - maxAge or token.lastUse < time - maxLastUse
|
||||
|
||||
proc isLimited(token: Token; api: Api): bool =
|
||||
if token.isNil or token.expired:
|
||||
return true
|
||||
|
||||
if api in token.apis:
|
||||
let limit = token.apis[api]
|
||||
return (limit.remaining <= 10 and limit.reset > epochTime().int)
|
||||
else:
|
||||
return false
|
||||
|
||||
proc isReady(token: Token; api: Api): bool =
|
||||
not (token.isNil or token.pending > maxConcurrentReqs or token.isLimited(api))
|
||||
|
||||
proc release*(token: Token; used=false; invalid=false) =
|
||||
if token.isNil: return
|
||||
if invalid or token.expired:
|
||||
if invalid: log "discarding invalid token"
|
||||
elif token.expired: log "discarding expired token"
|
||||
|
||||
let idx = tokenPool.find(token)
|
||||
if idx > -1: tokenPool.delete(idx)
|
||||
elif used:
|
||||
dec token.pending
|
||||
token.lastUse = getTime()
|
||||
|
||||
proc getToken*(api: Api): Future[Token] {.async.} =
|
||||
for i in 0 ..< tokenPool.len:
|
||||
if result.isReady(api): break
|
||||
release(result)
|
||||
result = tokenPool.sample()
|
||||
|
||||
if not result.isReady(api):
|
||||
release(result)
|
||||
result = await fetchToken()
|
||||
log "added new token to pool"
|
||||
tokenPool.add result
|
||||
|
||||
if not result.isNil:
|
||||
inc result.pending
|
||||
else:
|
||||
raise rateLimitError()
|
||||
|
||||
proc setRateLimit*(token: Token; api: Api; remaining, reset: int) =
|
||||
# avoid undefined behavior in race conditions
|
||||
if api in token.apis:
|
||||
let limit = token.apis[api]
|
||||
if limit.reset >= reset and limit.remaining < remaining:
|
||||
return
|
||||
|
||||
token.apis[api] = RateLimit(remaining: remaining, reset: reset)
|
||||
|
||||
proc poolTokens*(amount: int) {.async.} =
|
||||
var futs: seq[Future[Token]]
|
||||
for i in 0 ..< amount:
|
||||
futs.add fetchToken()
|
||||
|
||||
for token in futs:
|
||||
var newToken: Token
|
||||
|
||||
try: newToken = await token
|
||||
except: discard
|
||||
|
||||
if not newToken.isNil:
|
||||
log "added new token to pool"
|
||||
tokenPool.add newToken
|
||||
|
||||
proc initTokenPool*(cfg: Config) {.async.} =
|
||||
enableLogging = cfg.enableDebug
|
||||
|
||||
while true:
|
||||
if tokenPool.countIt(not it.isLimited(Api.userTweets)) < cfg.minTokens:
|
||||
await poolTokens(min(4, cfg.minTokens - tokenPool.len))
|
||||
await sleepAsync(2000)
|
||||
@ -12,25 +12,13 @@ type
|
||||
TimelineKind* {.pure.} = enum
|
||||
tweets, replies, media
|
||||
|
||||
Api* {.pure.} = enum
|
||||
tweetDetail
|
||||
tweetResult
|
||||
photoRail
|
||||
search
|
||||
list
|
||||
listBySlug
|
||||
listMembers
|
||||
listTweets
|
||||
userRestId
|
||||
userScreenName
|
||||
favorites
|
||||
userTweets
|
||||
userTweetsAndReplies
|
||||
userMedia
|
||||
favoriters
|
||||
retweeters
|
||||
following
|
||||
followers
|
||||
ApiUrl* = object
|
||||
endpoint*: string
|
||||
params*: seq[(string, string)]
|
||||
|
||||
ApiReq* = object
|
||||
oauth*: ApiUrl
|
||||
cookie*: ApiUrl
|
||||
|
||||
RateLimit* = object
|
||||
remaining*: int
|
||||
@ -43,14 +31,14 @@ type
|
||||
init*: Time
|
||||
lastUse*: Time
|
||||
pending*: int
|
||||
apis*: Table[Api, RateLimit]
|
||||
apis*: Table[string, RateLimit]
|
||||
|
||||
GuestAccount* = ref object
|
||||
id*: int64
|
||||
oauthToken*: string
|
||||
oauthSecret*: string
|
||||
pending*: int
|
||||
apis*: Table[Api, RateLimit]
|
||||
apis*: Table[string, RateLimit]
|
||||
|
||||
Error* = enum
|
||||
null = 0
|
||||
@ -308,6 +296,7 @@ type
|
||||
enableDebug*: bool
|
||||
proxy*: string
|
||||
proxyAuth*: string
|
||||
disableTid*: bool
|
||||
cookieHeader*: string
|
||||
xCsrfToken*: string
|
||||
|
||||
|
||||
@ -26,7 +26,9 @@ proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
|
||||
template verifiedIcon*(user: User): untyped {.dirty.} =
|
||||
if user.verifiedType != VerifiedType.none:
|
||||
let lower = ($user.verifiedType).toLowerAscii()
|
||||
icon "ok", class=(&"verified-icon {lower}"), title=(&"Verified {lower} account")
|
||||
buildHtml(tdiv(class=(&"verified-icon {lower}"))):
|
||||
icon "circle", class="verified-icon-circle", title=(&"Verified {lower} account")
|
||||
icon "ok", class="verified-icon-check", title=(&"Verified {lower} account")
|
||||
else:
|
||||
text ""
|
||||
|
||||
|
||||
@ -2,6 +2,9 @@
|
||||
## SPDX-License-Identifier: AGPL-3.0-only
|
||||
#import strutils, xmltree, strformat, options, unicode
|
||||
#import ../types, ../utils, ../formatters, ../prefs
|
||||
## Snowflake ID cutoff for RSS GUID format transition
|
||||
## Corresponds to approximately December 14, 2025 UTC
|
||||
#const guidCutoff = 2000000000000000000'i64
|
||||
#
|
||||
#proc getTitle(tweet: Tweet; retweet: string): string =
|
||||
#if tweet.pinned: result = "Pinned: "
|
||||
@ -25,7 +28,25 @@
|
||||
#end proc
|
||||
#
|
||||
#proc getDescription(desc: string; cfg: Config): string =
|
||||
Twitter feed for: ${desc}. Generated by ${cfg.hostname}
|
||||
Twitter feed for: ${desc}. Generated by ${getUrlPrefix(cfg)}
|
||||
#end proc
|
||||
#
|
||||
#proc getTweetsWithPinned(profile: Profile): seq[Tweets] =
|
||||
#result = profile.tweets.content
|
||||
#if profile.pinned.isSome and result.len > 0:
|
||||
# let pinnedTweet = profile.pinned.get
|
||||
# var inserted = false
|
||||
# for threadIdx in 0 ..< result.len:
|
||||
# if not inserted:
|
||||
# for tweetIdx in 0 ..< result[threadIdx].len:
|
||||
# if result[threadIdx][tweetIdx].id < pinnedTweet.id:
|
||||
# result[threadIdx].insert(pinnedTweet, tweetIdx)
|
||||
# inserted = true
|
||||
# end if
|
||||
# end for
|
||||
# end if
|
||||
# end for
|
||||
#end if
|
||||
#end proc
|
||||
#
|
||||
#proc renderRssTweet(tweet: Tweet; cfg: Config): string =
|
||||
@ -38,23 +59,34 @@ Twitter feed for: ${desc}. Generated by ${cfg.hostname}
|
||||
<img src="${urlPrefix}${getPicUrl(photo.url)}" style="max-width:250px;" />
|
||||
# end for
|
||||
#elif tweet.video.isSome:
|
||||
<a href="${urlPrefix}${tweet.getLink}">
|
||||
<br>Video<br>
|
||||
<img src="${urlPrefix}${getPicUrl(get(tweet.video).thumb)}" style="max-width:250px;" />
|
||||
</a>
|
||||
#elif tweet.gif.isSome:
|
||||
# let thumb = &"{urlPrefix}{getPicUrl(get(tweet.gif).thumb)}"
|
||||
# let url = &"{urlPrefix}{getPicUrl(get(tweet.gif).url)}"
|
||||
<video poster="${thumb}" autoplay muted loop style="max-width:250px;">
|
||||
<source src="${url}" type="video/mp4"></video>
|
||||
##elif tweet.card.isSome:
|
||||
## let card = tweet.card.get()
|
||||
## if card.image.len > 0:
|
||||
##<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
|
||||
## end if
|
||||
#elif tweet.card.isSome:
|
||||
# let card = tweet.card.get()
|
||||
# if card.image.len > 0:
|
||||
<img src="${urlPrefix}${getPicUrl(card.image)}" style="max-width:250px;" />
|
||||
# end if
|
||||
#end if
|
||||
#if tweet.quote.isSome and get(tweet.quote).available:
|
||||
# let quoteLink = getLink(get(tweet.quote))
|
||||
# let quoteTweet = get(tweet.quote)
|
||||
# let quoteLink = urlPrefix & getLink(quoteTweet)
|
||||
<hr/>
|
||||
<p>Quoting: <a href="${urlPrefix}${quoteLink}">${cfg.hostname}${quoteLink}</a></p>
|
||||
${renderRssTweet(get(tweet.quote), cfg)}
|
||||
<blockquote>
|
||||
<b>${quoteTweet.user.fullname} (@${quoteTweet.user.username})</b>
|
||||
<p>
|
||||
${renderRssTweet(quoteTweet, cfg)}
|
||||
</p>
|
||||
<footer>
|
||||
— <cite><a href="${quoteLink}">${quoteLink}</a>
|
||||
</footer>
|
||||
</blockquote>
|
||||
#end if
|
||||
#end proc
|
||||
#
|
||||
@ -72,12 +104,17 @@ ${renderRssTweet(get(tweet.quote), cfg)}
|
||||
# if link in links: continue
|
||||
# end if
|
||||
# links.add link
|
||||
# let useGlobalGuid = parseBiggestInt(tweet.id) >= guidCutoff
|
||||
<item>
|
||||
<title>${getTitle(tweet, retweet)}</title>
|
||||
<dc:creator>@${tweet.user.username}</dc:creator>
|
||||
<description><![CDATA[${renderRssTweet(tweet, cfg).strip(chars={'\n'})}]]></description>
|
||||
<pubDate>${getRfc822Time(tweet)}</pubDate>
|
||||
#if useGlobalGuid:
|
||||
<guid isPermaLink="false">${tweet.id}</guid>
|
||||
#else:
|
||||
<guid>${urlPrefix & link}</guid>
|
||||
#end if
|
||||
<link>${urlPrefix & link}</link>
|
||||
</item>
|
||||
# end for
|
||||
@ -108,8 +145,9 @@ ${renderRssTweet(get(tweet.quote), cfg)}
|
||||
<width>128</width>
|
||||
<height>128</height>
|
||||
</image>
|
||||
#if profile.tweets.content.len > 0:
|
||||
${renderRssTweets(profile.tweets.content, cfg, userId=profile.user.id)}
|
||||
#let tweetsList = getTweetsWithPinned(profile)
|
||||
#if tweetsList.len > 0:
|
||||
${renderRssTweets(tweetsList, cfg, userId=profile.user.id)}
|
||||
#end if
|
||||
</channel>
|
||||
</rss>
|
||||
|
||||
@ -28,13 +28,18 @@ proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode =
|
||||
if thread.hasMore:
|
||||
renderMoreReplies(thread)
|
||||
|
||||
proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode =
|
||||
proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string; tweet: Tweet = nil): VNode =
|
||||
buildHtml(tdiv(class="replies", id="r")):
|
||||
var hasReplies = false
|
||||
var replyCount = 0
|
||||
for thread in replies.content:
|
||||
if thread.content.len == 0: continue
|
||||
hasReplies = true
|
||||
replyCount += thread.content.len
|
||||
renderReplyThread(thread, prefs, path)
|
||||
|
||||
if replies.bottom.len > 0:
|
||||
if hasReplies and replies.bottom.len > 0:
|
||||
if tweet == nil or not replies.beginning or replyCount < tweet.stats.replies:
|
||||
renderMore(Query(), replies.bottom, focus="#r")
|
||||
|
||||
proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode =
|
||||
@ -70,6 +75,6 @@ proc renderConversation*(conv: Conversation; prefs: Prefs; path: string): VNode
|
||||
if not conv.replies.beginning:
|
||||
renderNewer(Query(), getLink(conv.tweet), focus="#r")
|
||||
if conv.replies.content.len > 0 or conv.replies.bottom.len > 0:
|
||||
renderReplies(conv.replies, prefs, path)
|
||||
renderReplies(conv.replies, prefs, path, conv.tweet)
|
||||
|
||||
renderToTop(focus="#m")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user