diff --git a/.rgignore b/.rgignore new file mode 100644 index 0000000..8e0ef12 --- /dev/null +++ b/.rgignore @@ -0,0 +1,2 @@ +pnpm-lock.yaml +public/fonts/ diff --git a/src/api.nim b/src/api.nim index 7f29791..886fee6 100644 --- a/src/api.nim +++ b/src/api.nim @@ -159,10 +159,9 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} = variables = %*{ "rawQuery": q, "count": 20, + "query_source": "typed_query", "product": "Latest", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false + "withGrokTranslatedBio": false } if after.len > 0: variables["cursor"] = % after @@ -178,10 +177,9 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} variables = %*{ "rawQuery": query.text, "count": 20, + "query_source": "typed_query", "product": "People", - "withDownvotePerspective": false, - "withReactionsMetadata": false, - "withReactionsPerspective": false + "withGrokTranslatedBio": false } if after.len > 0: variables["cursor"] = % after diff --git a/src/apiutils.nim b/src/apiutils.nim index 33454a7..0d038ed 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -36,13 +36,16 @@ proc genHeaders*(): HttpHeaders = "connection": "keep-alive", "authorization": auth, "content-type": "application/json", - #"x-guest-token": if token == nil: "" else: token.tok, - "x-twitter-active-user": "yes", - "authority": "api.twitter.com", "accept-encoding": "gzip", - "accept-language": "en-US,en;q=0.9", + "accept-language": "en-US,en;q=0.5", "accept": "*/*", - "DNT": "1" + "DNT": "1", + "Host": "x.com", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + "Sec-GPC": "1", + "TE": "trailers" }) #template updateToken() = @@ -62,9 +65,16 @@ template fetchImpl(result, additional_headers, fetchBody) {.dirty.} = if len(cfg.cookieHeader) != 0: additional_headers.add("Cookie", cfg.cookieHeader) + + additional_headers.add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0") + if len(cfg.xCsrfToken) != 0: additional_headers.add("x-csrf-token", cfg.xCsrfToken) + additional_headers.add("x-twitter-active-user", "yes") + additional_headers.add("x-twitter-auth-type", "OAuth2Session") + additional_headers.add("x-twitter-client-language", "en") + try: var resp: AsyncResponse #var headers = genHeaders(token) diff --git a/src/consts.nim b/src/consts.nim index 8132e7e..c8982aa 100644 --- a/src/consts.nim +++ b/src/consts.nim @@ -2,12 +2,12 @@ import uri, sequtils, strutils const - auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF" + auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" consumerKey* = "3nVuSoBZnx6U4vzUxf5w" consumerSecret* = "Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys" - api = parseUri("https://api.twitter.com") + api = parseUri("https://x.com/i/api") activate* = $(api / "1.1/guest/activate.json") photoRail* = api / "1.1/statuses/media_timeline.json" @@ -15,22 +15,23 @@ const timelineApi = api / "2/timeline" graphql = api / "graphql" - graphUser* = graphql / "32pL5BWe9WKeSK1MoPvFQQ/UserByScreenName" - graphUserById* = graphql / "5vdJ5sWkbSRDiiNZvwc2Yg/UserByRestId" - graphUserTweets* = graphql / "M3Hpkrb8pjWkEuGdLeXMOA/UserTweets" - graphUserTweetsAndReplies* = graphql / "pz0IHaV_t7T4HJavqqqcIA/UserTweetsAndReplies" - graphUserMedia* = graphql / "dexO_2tohK86JDudXXG3Yw/UserMedia" - graphTweet* = graphql / "b9Yw90FMr_zUb8DvA8r2ug/TweetDetail" - graphTweetResult* = graphql / "sITyJdhRPpvpEjg4waUmTA/TweetResultByIdQuery" - graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline" + graphUser* = graphql / "ZHSN3WlvahPKVvUxVQbg1A/UserByScreenName" + graphUserById* = graphql / "XIpMDIi_YoVzXeoON-cfAQ/UserByRestId" + graphUserTweets* = graphql / "oRJs8SLCRNRbQzuZG93_oA/UserTweets" + graphUserTweetsAndReplies* = graphql / "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies" + graphUserMedia* = graphql / "36oKqyQ7E_9CmtONGjJRsA/UserMedia" + graphTweet* = graphql / "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail" + graphTweetResult* = graphql / "tCVRZ3WCvoj0BVO7BKnL-Q/TweetResultByRestId" + graphTweetHistory* = graphql / "WT7HhrzWulh4yudKJaR10Q/TweetEditHistory" + graphSearchTimeline* = graphql / "7r8ibjHuK3MWUyzkzHNMYQ/SearchTimeline" graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId" graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug" graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers" graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline" graphFavoriters* = graphql / "mDc_nU8xGv0cLRWtTaIEug/Favoriters" graphRetweeters* = graphql / "RCR9gqwYD1NEgi9FWzA50A/Retweeters" - graphFollowers* = graphql / "EAqBhgcGr_qPOzhS4Q3scQ/Followers" - graphFollowing* = graphql / "JPZiqKjET7_M1r5Tlr8pyA/Following" + graphFollowers* = graphql / "Efm7xwLreAw77q2Fq7rX-Q/Followers" + graphFollowing* = graphql / "e0UtTAwQqgLKBllQxMgVxQ/Following" favorites* = graphql / "eSSNbhECHHWWALkkQq-YTA/Likes" timelineParams* = { @@ -50,66 +51,49 @@ const }.toSeq gqlFeatures* = """{ - "android_graphql_skip_api_media_color_palette": false, - "articles_preview_enabled": false, - "blue_business_profile_image_shape_enabled": false, - "c9s_tweet_anatomy_moderator_badge_enabled": false, - "communities_web_enable_tweet_community_results_fetch": false, - "creator_subscriptions_quote_tweet_preview_enabled": false, - "creator_subscriptions_subscription_count_enabled": false, "creator_subscriptions_tweet_preview_api_enabled": true, - "freedom_of_speech_not_reach_fetch_enabled": false, - "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false, - "hidden_profile_likes_enabled": false, - "highlights_tweets_tab_ui_enabled": false, - "interactive_text_enabled": false, - "longform_notetweets_consumption_enabled": true, - "longform_notetweets_inline_media_enabled": false, - "longform_notetweets_richtext_consumption_enabled": true, - "longform_notetweets_rich_text_read_enabled": false, - "responsive_web_edit_tweet_api_enabled": false, - "responsive_web_enhance_cards_enabled": false, - "responsive_web_graphql_exclude_directive_enabled": true, - "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, - "responsive_web_graphql_timeline_navigation_enabled": false, - "responsive_web_media_download_video_enabled": false, - "responsive_web_text_conversations_enabled": false, - "responsive_web_twitter_article_tweet_consumption_enabled": false, - "responsive_web_twitter_blue_verified_badge_is_enabled": true, - "rweb_lists_timeline_redesign_enabled": true, - "rweb_tipjar_consumption_enabled": false, - "rweb_video_timestamps_enabled": true, - "spaces_2022_h2_clipping": true, - "spaces_2022_h2_spaces_communities": true, - "standardized_nudges_misinfo": false, - "subscriptions_verification_info_enabled": true, - "subscriptions_verification_info_reason_enabled": true, - "subscriptions_verification_info_verified_since_enabled": true, - "super_follow_badge_privacy_enabled": false, - "super_follow_exclusive_tweet_notifications_enabled": false, - "super_follow_tweet_api_enabled": false, - "super_follow_user_api_enabled": false, - "tweet_awards_web_tipping_enabled": false, - "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, - "tweetypie_unmention_optimization_enabled": false, - "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false, - "verified_phone_label_enabled": false, - "vibe_api_enabled": false, - "view_counts_everywhere_api_enabled": true, "premium_content_api_read_enabled": false, + "communities_web_enable_tweet_community_results_fetch": true, + "c9s_tweet_anatomy_moderator_badge_enabled": true, "responsive_web_grok_analyze_button_fetch_trends_enabled": false, - "responsive_web_grok_analysis_button_from_backend": false, - "responsive_web_grok_analyze_post_followups_enabled": false, - "responsive_web_jetfuel_frame": false, + "responsive_web_grok_analyze_post_followups_enabled": true, + "responsive_web_jetfuel_frame": true, + "responsive_web_grok_share_attachment_enabled": true, + "articles_preview_enabled": true, + "responsive_web_edit_tweet_api_enabled": true, + "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true, + "view_counts_everywhere_api_enabled": true, + "longform_notetweets_consumption_enabled": true, + "responsive_web_twitter_article_tweet_consumption_enabled": true, + "tweet_awards_web_tipping_enabled": false, + "responsive_web_grok_show_grok_translated_post": true, + "responsive_web_grok_analysis_button_from_backend": true, + "creator_subscriptions_quote_tweet_preview_enabled": false, + "freedom_of_speech_not_reach_fetch_enabled": true, + "standardized_nudges_misinfo": true, + "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true, + "longform_notetweets_rich_text_read_enabled": true, + "longform_notetweets_inline_media_enabled": true, + "payments_enabled": false, "profile_label_improvements_pcf_label_in_post_enabled": true, - "responsive_web_grok_image_annotation_enabled": false, - "responsive_web_grok_share_attachment_enabled": false, - "rweb_video_screen_enabled": false, - "responsive_web_twitter_article_notes_tab_enabled": false, - "subscriptions_feature_can_gift_premium": false, + "responsive_web_profile_redirect_enabled": false, + "rweb_tipjar_consumption_enabled": true, + "verified_phone_label_enabled": false, + "responsive_web_grok_image_annotation_enabled": true, + "responsive_web_grok_imagine_annotation_enabled": true, + "responsive_web_grok_community_note_auto_translation_is_enabled": false, + "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false, + "responsive_web_graphql_timeline_navigation_enabled": true, + "responsive_web_enhance_cards_enabled": false, + "rweb_client_transaction_id_enabled": false, + "rweb_xp_forwarded_for_enabled": false, + "subscriptions_verification_info_verified_since_enabled": true, + "highlights_tweets_tab_ui_enabled": true, "hidden_profile_subscriptions_enabled": true, "subscriptions_verification_info_is_identity_verified_enabled": true, - "responsive_web_grok_show_grok_translated_post": false + "subscriptions_feature_can_gift_premium": false, + "responsive_web_twitter_article_notes_tab_enabled": true, + "rweb_video_screen_enabled": true }""".replace(" ", "").replace("\n", "") tweetVariables* = """{ @@ -125,8 +109,8 @@ const }""".replace(" ", "").replace("\n", "") tweetFieldToggles* = """{ - "withArticleRichContentState": false, - "withArticlePlainText": true, + "withArticleRichContentState": true, + "withArticlePlainText": false, "withGrokAnalyze": false, "withDisallowedReplyControls": false }""".replace(" ", "").replace("\n", "") @@ -137,7 +121,6 @@ const "count": 20, "includePromotedContent": false, "withCommunity": true, - "withQuickPromoteEligibilityTweetFields": false, "withVoice": true }""".replace(" ", "").replace("\n", "") @@ -165,5 +148,13 @@ const "userId": "$1", $2 "count": 20, - "includePromotedContent": false + "includePromotedContent": false, + "withClientEventToken": false, + "withBirdwatchNotes": true, + "withVoice": true +}""".replace(" ", "").replace("\n", "") + + tweetHistoryVariables* = """{ + "tweetId": "$1", + "withQuickPromoteEligibilityTweetFields": false }""".replace(" ", "").replace("\n", "") diff --git a/src/parser.nim b/src/parser.nim index e1cbb71..a59051b 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -2,7 +2,7 @@ import strutils, options, times, math import packedjson, packedjson/deserialiser import types, parserutils, utils -import experimental/parser/unifiedcard +import experimental/parser/[unifiedcard, utils] import std/tables proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet @@ -24,7 +24,7 @@ proc parseUser(js: JsonNode; id=""): User = media: js{"media_count"}.getInt, verifiedType: parseEnum[VerifiedType](js{"verified_type"}.getStr("None")), protected: js{"protected"}.getBool, - joinDate: js{"created_at"}.getTime + joinDate: parseTwitterDate(js{"created_at"}.getStr) ) result.expandUserEntities(js) @@ -40,7 +40,12 @@ proc parseGraphUser*(js: JsonNode): User = return User(suspended: true) result = parseUser(user{"legacy"}, user{"rest_id"}.getStr) - + + result.username = user{"core", "screen_name"}.getStr + result.fullname = user{"core", "name"}.getStr + result.joinDate = parseTwitterDate(user{"core", "created_at"}.getStr) + result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "") + let label = user{"affiliates_highlighted_label", "label"} if not label.isNull: let labelType = label{"userLabelType"}.getStr