+
+
+ END_HTML
+ end
+
+ if !thin_mode
+ author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).request_target}"
+ else
+ author_thumbnail = ""
+ end
+
+ author_name = HTML.escape(child["author"].as_s)
+ if child["verified"]?.try &.as_bool && child["authorIsChannelOwner"]?.try &.as_bool
+ author_name += " "
+ elsif child["verified"]?.try &.as_bool
+ author_name += " "
+ end
+ html << <<-END_HTML
+
+
+
+
+
+ END_HTML
+ end
+
+ if comments["continuation"]?
+ html << <<-END_HTML
+
+ 
+
+
+
+ + + #{author_name} + +
#{child["contentHtml"]}
+ END_HTML + + if child["attachment"]? + attachment = child["attachment"] + + case attachment["type"] + when "image" + attachment = attachment["imageThumbnails"][1] + + html << <<-END_HTML +
+
+ END_HTML
+ when "video"
+ html << <<-END_HTML
+
+ 
+
+
+
+
+ END_HTML
+ else nil # Ignore
+ end
+ end
+
+ html << <<-END_HTML
+ #{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}
+ |
+ END_HTML
+
+ if comments["videoId"]?
+ html << <<-END_HTML
+ [YT]
+ |
+ END_HTML
+ elsif comments["authorId"]?
+ html << <<-END_HTML
+ [YT]
+ |
+ END_HTML
+ end
+
+ html << <<-END_HTML
+ #{number_with_separator(child["likeCount"])}
+ END_HTML
+
+ if child["creatorHeart"]?
+ if !thin_mode
+ creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).request_target}"
+ else
+ creator_thumbnail = ""
+ end
+
+ html << <<-END_HTML
+
+
+
+
+ END_HTML
+
+ if attachment["error"]?
+ html << <<-END_HTML
+
+ #{attachment["error"]}
+ END_HTML + else + html << <<-END_HTML + + END_HTML + end + + html << <<-END_HTML +
+ 
+
+
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
+ #{replies_html}
+
+
+
+
+
+
+ END_HTML
+ end
+ end
+end
+
+def template_reddit_comments(root, locale)
+ String.build do |html|
+ root.each do |child|
+ if child.data.is_a?(RedditComment)
+ child = child.data.as(RedditComment)
+ body_html = HTML.unescape(child.body_html)
+
+ replies_html = ""
+ if child.replies.is_a?(RedditThing)
+ replies = child.replies.as(RedditThing)
+ replies_html = template_reddit_comments(replies.data.as(RedditListing).children, locale)
+ end
+
+ if child.depth > 0
+ html << <<-END_HTML
+
+
+
+
+ END_HTML
+ else
+ html << <<-END_HTML
+
") } || "" +end + +def content_to_comment_html(content, video_id : String? = "") + html_array = content.map do |run| + # Sometimes, there is an empty element. + # See: https://github.com/iv-org/invidious/issues/3096 + next if run.as_h.empty? + + text = HTML.escape(run["text"].as_s) + + if run["navigationEndpoint"]? + if url = run["navigationEndpoint"]["urlEndpoint"]?.try &.["url"].as_s + url = URI.parse(url) + displayed_url = text + + if url.host == "youtu.be" + url = "/watch?v=#{url.request_target.lstrip('/')}" + elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") + if url.path == "/redirect" + # Sometimes, links can be corrupted (why?) so make sure to fallback + # nicely. See https://github.com/iv-org/invidious/issues/2682 + url = url.query_params["q"]? || "" + displayed_url = url + else + url = url.request_target + displayed_url = "youtube.com#{url}" + end + end + + text = %(#{reduce_uri(displayed_url)}) + elsif watch_endpoint = run["navigationEndpoint"]["watchEndpoint"]? + start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i + link_video_id = watch_endpoint["videoId"].as_s + + url = "/watch?v=#{link_video_id}" + url += "&t=#{start_time}" if !start_time.nil? + + # If the current video ID (passed through from the caller function) + # is the same as the video ID in the link, add HTML attributes for + # the JS handler function that bypasses page reload. + # + # See: https://github.com/iv-org/invidious/issues/3063 + if link_video_id == video_id + start_time ||= 0 + text = %(#{reduce_uri(text)}) + else + text = %(#{text}) + end + elsif url = run.dig?("navigationEndpoint", "commandMetadata", "webCommandMetadata", "url").try &.as_s + if text.starts_with?(/\s?[@#]/) + # Handle "pings" in comments and hasthags differently + # See: + # - https://github.com/iv-org/invidious/issues/3038 + # - https://github.com/iv-org/invidious/issues/3062 + text = %(#{text}) + else + text = %(#{reduce_uri(url)}) + end + end + end + + text = "#{text}" if run["bold"]? + text = "#{text}" if run["italics"]? + + text + end + + return html_array.join("").delete('\ufeff') +end + +def produce_comment_continuation(video_id, cursor = "", sort_by = "top") + object = { + "2:embedded" => { + "2:string" => video_id, + "25:varint" => 0_i64, + "28:varint" => 1_i64, + "36:embedded" => { + "5:varint" => -1_i64, + "8:varint" => 0_i64, + }, + "40:embedded" => { + "1:varint" => 4_i64, + "3:string" => "https://www.youtube.com", + "4:string" => "", + }, + }, + "3:varint" => 6_i64, + "6:embedded" => { + "1:string" => cursor, + "4:embedded" => { + "4:string" => video_id, + "6:varint" => 0_i64, + }, + "5:varint" => 20_i64, + }, + } + + case sort_by + when "top" + object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 + when "new", "newest" + object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64 + else # top + object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 + end + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr deleted file mode 100644 index 1f55bfe6..00000000 --- a/src/invidious/comments/content.cr +++ /dev/null @@ -1,89 +0,0 @@ -def text_to_parsed_content(text : String) : JSON::Any - nodes = [] of JSON::Any - # For each line convert line to array of nodes - text.split('\n').each do |line| - # In first case line is just a simple node before - # check patterns inside line - # { 'text': line } - current_nodes = [] of JSON::Any - initial_node = {"text" => line} - current_nodes << (JSON.parse(initial_node.to_json)) - - # For each match with url pattern, get last node and preserve - # last node before create new node with url information - # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } - line.scan(/https?:\/\/[^ ]*/).each do |url_match| - # Retrieve last node and update node without match - last_node = current_nodes[-1].as_h - splitted_last_node = last_node["text"].as_s.split(url_match[0]) - last_node["text"] = JSON.parse(splitted_last_node[0].to_json) - current_nodes[-1] = JSON.parse(last_node.to_json) - # Create new node with match and navigation infos - current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}} - current_nodes << (JSON.parse(current_node.to_json)) - # If text remain after match create new simple node with text after match - after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""} - current_nodes << (JSON.parse(after_node.to_json)) - end - - # After processing of matches inside line - # Add \n at end of last node for preserve carriage return - last_node = current_nodes[-1].as_h - last_node["text"] = JSON.parse("#{last_node["text"]}\n".to_json) - current_nodes[-1] = JSON.parse(last_node.to_json) - - # Finally add final nodes to nodes returned - current_nodes.each do |node| - nodes << (node) - end - end - return JSON.parse({"runs" => nodes}.to_json) -end - -def parse_content(content : JSON::Any, video_id : String? = "") : String - content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || - content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r, video_id).try &.to_s.gsub("\n", "
") } || "" -end - -def content_to_comment_html(content, video_id : String? = "") - html_array = content.map do |run| - # Sometimes, there is an empty element. - # See: https://github.com/iv-org/invidious/issues/3096 - next if run.as_h.empty? - - text = HTML.escape(run["text"].as_s) - - if navigation_endpoint = run.dig?("navigationEndpoint") - text = parse_link_endpoint(navigation_endpoint, text, video_id) - end - - text = "#{text}" if run["bold"]? - text = "#{text}" if run["strikethrough"]?
- text = "#{text}" if run["italics"]?
-
- # check for custom emojis
- if run["emoji"]?
- if run["emoji"]["isCustomEmoji"]?.try &.as_bool
- if emoji_image = run.dig?("emoji", "image")
- emoji_alt = emoji_image.dig?("accessibility", "accessibilityData", "label").try &.as_s || text
- emoji_thumb = emoji_image["thumbnails"][0]
- text = String.build do |str|
- str << %(![) << emoji_alt <<]()
)
- end
- else
- # Hide deleted channel emoji
- text = ""
- end
- end
- end
-
- text
- end
-
- return html_array.join("").delete('\ufeff')
-end
diff --git a/src/invidious/comments/links_util.cr b/src/invidious/comments/links_util.cr
deleted file mode 100644
index f89b86d3..00000000
--- a/src/invidious/comments/links_util.cr
+++ /dev/null
@@ -1,76 +0,0 @@
-module Invidious::Comments
- extend self
-
- def replace_links(html)
- # Check if the document is empty
- # Prevents edge-case bug with Reddit comments, see issue #3115
- if html.nil? || html.empty?
- return html
- end
-
- html = XML.parse_html(html)
-
- html.xpath_nodes(%q(//a)).each do |anchor|
- url = URI.parse(anchor["href"])
-
- if url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") || url.host.not_nil!.ends_with?("youtu.be")
- if url.host.try &.ends_with? "youtu.be"
- url = "/watch?v=#{url.path.lstrip('/')}#{url.query_params}"
- else
- if url.path == "/redirect"
- params = HTTP::Params.parse(url.query.not_nil!)
- anchor["href"] = params["q"]?
- else
- anchor["href"] = url.request_target
- end
- end
- elsif url.to_s == "#"
- begin
- length_seconds = decode_length_seconds(anchor.content)
- rescue ex
- length_seconds = decode_time(anchor.content)
- end
-
- if length_seconds > 0
- anchor["href"] = "javascript:void(0)"
- anchor["onclick"] = "player.currentTime(#{length_seconds})"
- else
- anchor["href"] = url.request_target
- end
- end
- end
-
- html = html.xpath_node(%q(//body)).not_nil!
- if node = html.xpath_node(%q(./p))
- html = node
- end
-
- return html.to_xml(options: XML::SaveOptions::NO_DECL)
- end
-
- def fill_links(html, scheme, host)
- # Check if the document is empty
- # Prevents edge-case bug with Reddit comments, see issue #3115
- if html.nil? || html.empty?
- return html
- end
-
- html = XML.parse_html(html)
-
- html.xpath_nodes("//a").each do |match|
- url = URI.parse(match["href"])
- # Reddit links don't have host
- if !url.host && !match["href"].starts_with?("javascript") && !url.to_s.ends_with? "#"
- url.scheme = scheme
- url.host = host
- match["href"] = url
- end
- end
-
- if host == "www.youtube.com"
- html = html.xpath_node(%q(//body/p)).not_nil!
- end
-
- return html.to_xml(options: XML::SaveOptions::NO_DECL)
- end
-end
diff --git a/src/invidious/comments/reddit.cr b/src/invidious/comments/reddit.cr
deleted file mode 100644
index ba9c19f1..00000000
--- a/src/invidious/comments/reddit.cr
+++ /dev/null
@@ -1,41 +0,0 @@
-module Invidious::Comments
- extend self
-
- def fetch_reddit(id, sort_by = "confidence")
- client = make_client(REDDIT_URL)
- headers = HTTP::Headers{"User-Agent" => "web:invidious:v#{CURRENT_VERSION} (by github.com/iv-org/invidious)"}
-
- # TODO: Use something like #479 for a static list of instances to use here
- query = URI::Params.encode({q: "(url:3D#{id} OR url:#{id}) AND (site:invidio.us OR site:youtube.com OR site:youtu.be)"})
- search_results = client.get("/search.json?#{query}", headers)
-
- if search_results.status_code == 200
- search_results = RedditThing.from_json(search_results.body)
-
- # For videos that have more than one thread, choose the one with the highest score
- threads = search_results.data.as(RedditListing).children
- thread = threads.max_by?(&.data.as(RedditLink).score).try(&.data.as(RedditLink))
- result = thread.try do |t|
- body = client.get("/r/#{t.subreddit}/comments/#{t.id}.json?limit=100&sort=#{sort_by}", headers).body
- Array(RedditThing).from_json(body)
- end
- result ||= [] of RedditThing
- elsif search_results.status_code == 302
- # Previously, if there was only one result then the API would redirect to that result.
- # Now, it appears it will still return a listing so this section is likely unnecessary.
-
- result = client.get(search_results.headers["Location"], headers).body
- result = Array(RedditThing).from_json(result)
-
- thread = result[0].data.as(RedditListing).children[0].data.as(RedditLink)
- else
- raise NotFoundException.new("Comments not found.")
- end
-
- client.close
-
- comments = result[1]?.try(&.data.as(RedditListing).children)
- comments ||= [] of RedditThing
- return comments, thread
- end
-end
diff --git a/src/invidious/comments/reddit_types.cr b/src/invidious/comments/reddit_types.cr
deleted file mode 100644
index 796a1183..00000000
--- a/src/invidious/comments/reddit_types.cr
+++ /dev/null
@@ -1,57 +0,0 @@
-class RedditThing
- include JSON::Serializable
-
- property kind : String
- property data : RedditComment | RedditLink | RedditMore | RedditListing
-end
-
-class RedditComment
- include JSON::Serializable
-
- property author : String
- property body_html : String
- property replies : RedditThing | String
- property score : Int32
- property depth : Int32
- property permalink : String
-
- @[JSON::Field(converter: RedditComment::TimeConverter)]
- property created_utc : Time
-
- module TimeConverter
- def self.from_json(value : JSON::PullParser) : Time
- Time.unix(value.read_float.to_i)
- end
-
- def self.to_json(value : Time, json : JSON::Builder)
- json.number(value.to_unix)
- end
- end
-end
-
-struct RedditLink
- include JSON::Serializable
-
- property author : String
- property score : Int32
- property subreddit : String
- property num_comments : Int32
- property id : String
- property permalink : String
- property title : String
-end
-
-struct RedditMore
- include JSON::Serializable
-
- property children : Array(String)
- property count : Int32
- property depth : Int32
-end
-
-class RedditListing
- include JSON::Serializable
-
- property children : Array(RedditThing)
- property modhash : String
-end
diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr
deleted file mode 100644
index 0716fcde..00000000
--- a/src/invidious/comments/youtube.cr
+++ /dev/null
@@ -1,365 +0,0 @@
-module Invidious::Comments
- extend self
-
- def fetch_youtube(id, cursor, format, locale, thin_mode, region, sort_by = "top")
- case cursor
- when nil, ""
- ctoken = Comments.produce_continuation(id, cursor: "", sort_by: sort_by)
- when .starts_with? "ADSJ"
- ctoken = Comments.produce_continuation(id, cursor: cursor, sort_by: sort_by)
- else
- ctoken = cursor
- end
-
- client_config = YoutubeAPI::ClientConfig.new(region: region)
- response = YoutubeAPI.next(continuation: ctoken, client_config: client_config)
- return parse_youtube(id, response, format, locale, thin_mode, sort_by)
- end
-
- def fetch_community_post_comments(ucid, post_id)
- object = {
- "2:string" => "community",
- "25:embedded" => {
- "22:string" => post_id,
- },
- "45:embedded" => {
- "2:varint" => 1_i64,
- "3:varint" => 1_i64,
- },
- "53:embedded" => {
- "4:embedded" => {
- "6:varint" => 0_i64,
- "27:varint" => 1_i64,
- "29:string" => post_id,
- "30:string" => ucid,
- },
- "8:string" => "comments-section",
- },
- }
-
- object_parsed = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
-
- object2 = {
- "80226972:embedded" => {
- "2:string" => ucid,
- "3:string" => object_parsed,
- },
- }
-
- continuation = object2.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- initial_data = YoutubeAPI.browse(continuation: continuation)
- return initial_data
- end
-
- def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", is_post = false)
- contents = nil
-
- if on_response_received_endpoints = response["onResponseReceivedEndpoints"]?
- header = nil
- on_response_received_endpoints.as_a.each do |item|
- if item["reloadContinuationItemsCommand"]?
- case item["reloadContinuationItemsCommand"]["slot"]
- when "RELOAD_CONTINUATION_SLOT_HEADER"
- header = item["reloadContinuationItemsCommand"]["continuationItems"][0]
- when "RELOAD_CONTINUATION_SLOT_BODY"
- # continuationItems is nil when video has no comments
- contents = item["reloadContinuationItemsCommand"]["continuationItems"]?
- end
- elsif item["appendContinuationItemsAction"]?
- contents = item["appendContinuationItemsAction"]["continuationItems"]
- end
- end
- elsif response["continuationContents"]?
- response = response["continuationContents"]
- if response["commentRepliesContinuation"]?
- body = response["commentRepliesContinuation"]
- else
- body = response["itemSectionContinuation"]
- end
- contents = body["contents"]?
- header = body["header"]?
- else
- raise NotFoundException.new("Comments not found.")
- end
-
- if !contents
- if format == "json"
- return {"comments" => [] of String}.to_json
- else
- return {"contentHtml" => "", "commentCount" => 0}.to_json
- end
- end
-
- continuation_item_renderer = nil
- contents.as_a.reject! do |item|
- if item["continuationItemRenderer"]?
- continuation_item_renderer = item["continuationItemRenderer"]
- true
- end
- end
-
- mutations = response.dig?("frameworkUpdates", "entityBatchUpdate", "mutations").try &.as_a || [] of JSON::Any
-
- response = JSON.build do |json|
- json.object do
- if header
- count_text = header["commentsHeaderRenderer"]["countText"]
- comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?)
- .try &.as_s.gsub(/\D/, "").to_i? || 0
- json.field "commentCount", comment_count
- end
-
- if is_post
- json.field "postId", id
- else
- json.field "videoId", id
- end
-
- json.field "comments" do
- json.array do
- contents.as_a.each do |node|
- json.object do
- if node["commentThreadRenderer"]?
- node = node["commentThreadRenderer"]
- end
-
- if node["replies"]?
- node_replies = node["replies"]["commentRepliesRenderer"]
- end
-
- if cvm = node["commentViewModel"]?
- # two commentViewModels for inital request
- # one commentViewModel when getting a replies to a comment
- cvm = cvm["commentViewModel"] if cvm["commentViewModel"]?
-
- comment_key = cvm["commentKey"]
- toolbar_key = cvm["toolbarStateKey"]
- comment_mutation = mutations.find { |i| i.dig?("payload", "commentEntityPayload", "key") == comment_key }
- toolbar_mutation = mutations.find { |i| i.dig?("entityKey") == toolbar_key }
-
- if !comment_mutation.nil? && !toolbar_mutation.nil?
- # todo parse styleRuns, commandRuns and attachmentRuns for comments
- html_content = parse_description(comment_mutation.dig("payload", "commentEntityPayload", "properties", "content"), id)
- comment_author = comment_mutation.dig("payload", "commentEntityPayload", "author")
- json.field "authorId", comment_author["channelId"].as_s
- json.field "authorUrl", "/channel/#{comment_author["channelId"].as_s}"
- json.field "author", comment_author["displayName"].as_s
- json.field "verified", comment_author["isVerified"].as_bool
- json.field "authorThumbnails" do
- json.array do
- comment_mutation.dig?("payload", "commentEntityPayload", "avatar", "image", "sources").try &.as_a.each do |thumbnail|
- json.object do
- json.field "url", thumbnail["url"]
- json.field "width", thumbnail["width"]
- json.field "height", thumbnail["height"]
- end
- end
- end
- end
-
- json.field "authorIsChannelOwner", comment_author["isCreator"].as_bool
- json.field "isSponsor", (comment_author["sponsorBadgeUrl"]? != nil)
-
- if sponsor_badge_url = comment_author["sponsorBadgeUrl"]?
- # Sponsor icon thumbnails always have one object and there's only ever the url property in it
- json.field "sponsorIconUrl", sponsor_badge_url
- end
-
- comment_toolbar = comment_mutation.dig("payload", "commentEntityPayload", "toolbar")
- json.field "likeCount", short_text_to_number(comment_toolbar["likeCountNotliked"].as_s)
- reply_count = short_text_to_number(comment_toolbar["replyCount"]?.try &.as_s || "0")
-
- if heart_state = toolbar_mutation.dig?("payload", "engagementToolbarStateEntityPayload", "heartState")
- if heart_state.as_s == "TOOLBAR_HEART_STATE_HEARTED"
- json.field "creatorHeart" do
- json.object do
- json.field "creatorThumbnail", comment_toolbar["creatorThumbnailUrl"].as_s
- json.field "creatorName", comment_toolbar["heartActiveTooltip"].as_s.sub("❤ by ", "")
- end
- end
- end
- end
-
- published_text = comment_mutation.dig?("payload", "commentEntityPayload", "properties", "publishedTime").try &.as_s
- end
-
- json.field "isPinned", (cvm.dig?("pinnedText") != nil)
- json.field "commentId", cvm["commentId"]
- else
- if node["comment"]?
- node_comment = node["comment"]["commentRenderer"]
- else
- node_comment = node["commentRenderer"]
- end
- json.field "commentId", node_comment["commentId"]
- html_content = node_comment["contentText"]?.try { |t| parse_content(t, id) }
-
- json.field "verified", (node_comment["authorCommentBadge"]? != nil)
-
- json.field "author", node_comment["authorText"]?.try &.["simpleText"]? || ""
- json.field "authorThumbnails" do
- json.array do
- node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail|
- json.object do
- json.field "url", thumbnail["url"]
- json.field "width", thumbnail["width"]
- json.field "height", thumbnail["height"]
- end
- end
- end
- end
-
- if comment_action_buttons_renderer = node_comment.dig?("actionButtons", "commentActionButtonsRenderer")
- json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i
- if comment_action_buttons_renderer["creatorHeart"]?
- heart_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"]
- json.field "creatorHeart" do
- json.object do
- json.field "creatorThumbnail", heart_data["thumbnails"][-1]["url"]
- json.field "creatorName", heart_data["accessibility"]["accessibilityData"]["label"]
- end
- end
- end
- end
-
- if node_comment["authorEndpoint"]?
- json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"]
- json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"]
- else
- json.field "authorId", ""
- json.field "authorUrl", ""
- end
-
- json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"]
- json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil)
- published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s
-
- json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil)
- if node_comment["sponsorCommentBadge"]?
- # Sponsor icon thumbnails always have one object and there's only ever the url property in it
- json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s
- end
-
- reply_count = node_comment["replyCount"]?
- end
-
- content_html = html_content || ""
- json.field "content", html_to_content(content_html)
- json.field "contentHtml", content_html
-
- if published_text != nil
- published_text = published_text.to_s
- if published_text.includes?(" (edited)")
- json.field "isEdited", true
- published = decode_date(published_text.rchop(" (edited)"))
- else
- json.field "isEdited", false
- published = decode_date(published_text)
- end
-
- json.field "published", published.to_unix
- json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
- end
-
- if node_replies && !response["commentRepliesContinuation"]?
- if node_replies["continuations"]?
- continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
- elsif node_replies["contents"]?
- continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s
- end
- continuation ||= ""
-
- json.field "replies" do
- json.object do
- json.field "replyCount", reply_count || 1
- json.field "continuation", continuation
- end
- end
- end
- end
- end
- end
- end
-
- if continuation_item_renderer
- if continuation_item_renderer["continuationEndpoint"]?
- continuation_endpoint = continuation_item_renderer["continuationEndpoint"]
- elsif continuation_item_renderer["button"]?
- continuation_endpoint = continuation_item_renderer["button"]["buttonRenderer"]["command"]
- end
- if continuation_endpoint
- json.field "continuation", continuation_endpoint["continuationCommand"]["token"].as_s
- end
- end
- end
- end
-
- if format == "html"
- response = JSON.parse(response)
- content_html = Frontend::Comments.template_youtube(response, locale, thin_mode)
- response = JSON.build do |json|
- json.object do
- json.field "contentHtml", content_html
-
- if response["commentCount"]?
- json.field "commentCount", response["commentCount"]
- else
- json.field "commentCount", 0
- end
- end
- end
- end
-
- return response
- end
-
- def produce_continuation(video_id, cursor = "", sort_by = "top")
- object = {
- "2:embedded" => {
- "2:string" => video_id,
- "25:varint" => 0_i64,
- "28:varint" => 1_i64,
- "36:embedded" => {
- "5:varint" => -1_i64,
- "8:varint" => 0_i64,
- },
- "40:embedded" => {
- "1:varint" => 4_i64,
- "3:string" => "https://www.youtube.com",
- "4:string" => "",
- },
- },
- "3:varint" => 6_i64,
- "6:embedded" => {
- "1:string" => cursor,
- "4:embedded" => {
- "4:string" => video_id,
- "6:varint" => 0_i64,
- },
- "5:varint" => 20_i64,
- },
- }
-
- case sort_by
- when "top"
- object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
- when "new", "newest"
- object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64
- else # top
- object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
- end
-
- continuation = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- return continuation
- end
-end
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 4d69854c..786b65df 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -8,19 +8,11 @@ struct DBConfig
property dbname : String
end
-struct SocketBindingConfig
- include YAML::Serializable
-
- property path : String
- property permissions : String
-end
-
struct ConfigPreferences
include YAML::Serializable
property annotations : Bool = false
property annotations_subscribed : Bool = false
- property preload : Bool = true
property autoplay : Bool = false
property captions : Array(String) = ["", "", ""]
property comments : Array(String) = ["youtube", ""]
@@ -35,7 +27,7 @@ struct ConfigPreferences
property max_results : Int32 = 40
property notifications_only : Bool = false
property player_style : String = "invidious"
- property quality : String = "dash"
+ property quality : String = "hd720"
property quality_dash : String = "auto"
property default_home : String? = "Popular"
property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
@@ -56,34 +48,15 @@ struct ConfigPreferences
def to_tuple
{% begin %}
{
- {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}}
+ {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
}
{% end %}
end
end
-struct HTTPProxyConfig
- include YAML::Serializable
-
- property user : String
- property password : String
- property host : String
- property port : Int32
-end
-
class Config
include YAML::Serializable
- class CompanionConfig
- include YAML::Serializable
-
- @[YAML::Field(converter: Preferences::URIConverter)]
- property private_url : URI = URI.parse("")
-
- @[YAML::Field(converter: Preferences::URIConverter)]
- property public_url : URI = URI.parse("")
- end
-
# Number of threads to use for crawling videos from channels (for updating subscriptions)
property channel_threads : Int32 = 1
# Time interval between two executions of the job that crawls channel videos (subscriptions update).
@@ -95,24 +68,20 @@ class Config
property output : String = "STDOUT"
# Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
property log_level : LogLevel = LogLevel::Info
- # Enables colors in logs. Useful for debugging purposes
- property colorize_logs : Bool = false
# Database configuration with separate parameters (username, hostname, etc)
property db : DBConfig? = nil
# Database configuration using 12-Factor "Database URL" syntax
@[YAML::Field(converter: Preferences::URIConverter)]
property database_url : URI = URI.parse("")
+ # Use polling to keep decryption function up to date
+ property decrypt_polling : Bool = false
# Used for crawling channels: threads should check all videos uploaded by a channel
property full_refresh : Bool = false
-
- # Jobs config structure. See jobs.cr and jobs/base_job.cr
- property jobs = Invidious::Jobs::JobsConfig.new
-
# Used to tell Invidious it is behind a proxy, so links to resources should be https://
property https_only : Bool?
# HMAC signing key for CSRF tokens and verifying pubsub subscriptions
- property hmac_key : String = ""
+ property hmac_key : String?
# Domain to be used for links to resources on the site where an absolute URL is required
property domain : String?
# Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
@@ -137,8 +106,6 @@ class Config
property hsts : Bool? = true
# Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
property disable_proxy : Bool? | Array(String)? = false
- # Enable the user notifications for all users
- property enable_user_notifications : Bool = true
# URL to the modified source code to be easily AGPL compliant
# Will display in the footer, next to the main source code link
@@ -147,41 +114,22 @@ class Config
# Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
@[YAML::Field(converter: Preferences::FamilyConverter)]
property force_resolve : Socket::Family = Socket::Family::UNSPEC
-
- # External signature solver server socket (either a path to a UNIX domain socket or ":")
- property signature_server : String? = nil
-
# Port to listen for connections (overridden by command line argument)
property port : Int32 = 3000
# Host to bind (overridden by command line argument)
property host_binding : String = "0.0.0.0"
- # Path and permissions to make Invidious listen on a UNIX socket instead of a TCP port
- property socket_binding : SocketBindingConfig? = nil
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100
- # HTTP Proxy configuration
- property http_proxy : HTTPProxyConfig? = nil
-
- # Use Innertube's transcripts API instead of timedtext for closed captions
- property use_innertube_for_captions : Bool = false
-
- # visitor data ID for Google session
- property visitor_data : String? = nil
- # poToken for passing bot attestation
- property po_token : String? = nil
-
- # Invidious companion
- property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig
-
- # Invidious companion API key
- property invidious_companion_key : String = ""
+ # Use quic transport for youtube api
+ property use_quic : Bool = false
# Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new
-
- # Playlist length limit
- property playlist_length_limit : Int32 = 500
+ # Key for Anti-Captcha
+ property captcha_key : String? = nil
+ # API URL for Anti-Captcha
+ property captcha_api_url : String = "https://api.anti-captcha.com"
def disabled?(option)
case disabled = CONFIG.disable_proxy
@@ -209,9 +157,6 @@ class Config
config = Config.from_yaml(config_yaml)
# Update config from env vars (upcased and prefixed with "INVIDIOUS_")
- #
- # Also checks if any top-level config options are set to "CHANGE_ME!!"
- # TODO: Support non-top-level config options such as the ones in DBConfig
{% for ivar in Config.instance_vars %}
{% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
@@ -248,42 +193,8 @@ class Config
exit(1)
end
end
-
- # Warn when any config attribute is set to "CHANGE_ME!!"
- if config.{{ivar.id}} == "CHANGE_ME!!"
- puts "Config: The value of '#{ {{ivar.stringify}} }' needs to be changed!!"
- exit(1)
- end
{% end %}
- if config.invidious_companion.present?
- # invidious_companion and signature_server can't work together
- if config.signature_server
- puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."
- exit(1)
- elsif config.invidious_companion_key.empty?
- puts "Config: Please configure a key if you are using invidious companion."
- exit(1)
- elsif config.invidious_companion_key == "CHANGE_ME!!"
- puts "Config: The value of 'invidious_companion_key' needs to be changed!!"
- exit(1)
- elsif config.invidious_companion_key.size != 16
- puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 characters."
- exit(1)
- end
- elsif config.signature_server
- puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/companion-installation/")
- else
- puts("WARNING: Invidious companion is required to view and playback videos. For more information see https://docs.invidious.io/companion-installation/")
- end
-
- # HMAC_key is mandatory
- # See: https://github.com/iv-org/invidious/issues/3854
- if config.hmac_key.empty?
- puts "Config: 'hmac_key' is required/can't be empty"
- exit(1)
- end
-
# Build database_url from db.* if it's not set directly
if config.database_url.to_s.empty?
if db = config.db
@@ -296,25 +207,7 @@ class Config
path: db.dbname,
)
else
- puts "Config: Either database_url or db.* is required"
- exit(1)
- end
- end
-
- # Check if the socket configuration is valid
- if sb = config.socket_binding
- if sb.path.ends_with?("/") || File.directory?(sb.path)
- puts "Config: The socket path " + sb.path + " must not be a directory!"
- exit(1)
- end
- d = File.dirname(sb.path)
- if !File.directory?(d)
- puts "Config: Socket directory " + sb.path + " does not exist or is not a directory!"
- exit(1)
- end
- p = sb.permissions.to_i?(base: 8)
- if !p || p < 0 || p > 0o777
- puts "Config: Socket permissions must be an octal between 0 and 777!"
+ puts "Config : Either database_url or db.* is required"
exit(1)
end
end
diff --git a/src/invidious/database/nonces.cr b/src/invidious/database/nonces.cr
index b87c81ec..469fcbd8 100644
--- a/src/invidious/database/nonces.cr
+++ b/src/invidious/database/nonces.cr
@@ -4,7 +4,7 @@ module Invidious::Database::Nonces
extend self
# -------------------
- # Insert / Delete
+ # Insert
# -------------------
def insert(nonce : String, expire : Time)
@@ -17,15 +17,6 @@ module Invidious::Database::Nonces
PG_DB.exec(request, nonce, expire)
end
- def delete_expired
- request = <<-SQL
- DELETE FROM nonces *
- WHERE expire < now()
- SQL
-
- PG_DB.exec(request)
- end
-
# -------------------
# Update
# -------------------
diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr
index 6dbcaa05..c6754a1e 100644
--- a/src/invidious/database/playlists.cr
+++ b/src/invidious/database/playlists.cr
@@ -91,7 +91,7 @@ module Invidious::Database::Playlists
end
# -------------------
- # Select
+ # Salect
# -------------------
def select(*, id : String) : InvidiousPlaylist?
@@ -113,7 +113,7 @@ module Invidious::Database::Playlists
end
# -------------------
- # Select (filtered)
+ # Salect (filtered)
# -------------------
def select_like_iv(email : String) : Array(InvidiousPlaylist)
@@ -140,7 +140,6 @@ module Invidious::Database::Playlists
request = <<-SQL
SELECT id,title FROM playlists
WHERE author = $1 AND id LIKE 'IV%'
- ORDER BY title
SQL
PG_DB.query_all(request, email, as: {String, String})
@@ -213,7 +212,7 @@ module Invidious::Database::PlaylistVideos
end
# -------------------
- # Select
+ # Salect
# -------------------
def select(plid : String, index : VideoIndex, offset, limit = 100) : Array(PlaylistVideo)
diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr
index 9e4963fd..1df549e2 100644
--- a/src/invidious/database/statistics.cr
+++ b/src/invidious/database/statistics.cr
@@ -15,7 +15,7 @@ module Invidious::Database::Statistics
PG_DB.query_one(request, as: Int64)
end
- def count_users_active_6m : Int64
+ def count_users_active_1m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '6 months'
@@ -24,7 +24,7 @@ module Invidious::Database::Statistics
PG_DB.query_one(request, as: Int64)
end
- def count_users_active_1m : Int64
+ def count_users_active_6m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '1 month'
diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr
index 4a3056ea..f62b43ea 100644
--- a/src/invidious/database/users.cr
+++ b/src/invidious/database/users.cr
@@ -52,7 +52,7 @@ module Invidious::Database::Users
def mark_watched(user : User, vid : String)
request = <<-SQL
UPDATE users
- SET watched = array_append(array_remove(watched, $1), $1)
+ SET watched = array_append(watched, $1)
WHERE email = $2
SQL
@@ -119,15 +119,15 @@ module Invidious::Database::Users
# Update (notifs)
# -------------------
- def add_multiple_notifications(channel_id : String, video_ids : Array(String))
+ def add_notification(video : ChannelVideo)
request = <<-SQL
UPDATE users
- SET notifications = array_cat(notifications, $1),
+ SET notifications = array_append(notifications, $1),
feed_needs_update = true
WHERE $2 = ANY(subscriptions)
SQL
- PG_DB.exec(request, video_ids, channel_id)
+ PG_DB.exec(request, video.id, video.ucid)
end
def remove_notification(user : User, vid : String)
@@ -154,16 +154,6 @@ module Invidious::Database::Users
# Update (misc)
# -------------------
- def feed_needs_update(channel_id : String)
- request = <<-SQL
- UPDATE users
- SET feed_needs_update = true
- WHERE $1 = ANY(subscriptions)
- SQL
-
- PG_DB.exec(request, channel_id)
- end
-
def update_preferences(user : User)
request = <<-SQL
UPDATE users
diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr
index 695f5b33..e1fa01c3 100644
--- a/src/invidious/database/videos.cr
+++ b/src/invidious/database/videos.cr
@@ -22,15 +22,6 @@ module Invidious::Database::Videos
PG_DB.exec(request, id)
end
- def delete_expired
- request = <<-SQL
- DELETE FROM videos *
- WHERE updated < (now() - interval '6 hours')
- SQL
-
- PG_DB.exec(request)
- end
-
def update(video : Video)
request = <<-SQL
UPDATE videos
diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr
index 690db907..425c08da 100644
--- a/src/invidious/exceptions.cr
+++ b/src/invidious/exceptions.cr
@@ -33,8 +33,3 @@ end
class VideoNotAvailableException < Exception
end
-
-# Exception used to indicate that the JSON response from YT is missing
-# some important informations, and that the query should be sent again.
-class RetryOnceException < Exception
-end
diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr
deleted file mode 100644
index 4fe21b96..00000000
--- a/src/invidious/frontend/channel_page.cr
+++ /dev/null
@@ -1,47 +0,0 @@
-module Invidious::Frontend::ChannelPage
- extend self
-
- enum TabsAvailable
- Videos
- Shorts
- Streams
- Podcasts
- Releases
- Courses
- Playlists
- Posts
- Channels
- end
-
- def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
- return String.build(1500) do |str|
- base_url = "/channel/#{channel.ucid}"
-
- TabsAvailable.each do |tab|
- # Ignore playlists, as it is not supported for auto-generated channels yet
- next if (tab.playlists? && channel.auto_generated)
-
- tab_name = tab.to_s.downcase
-
- if channel.tabs.includes? tab_name
- str << %(
+
+ END_HTML
+ end
+ end
+ end
+end
+
+def replace_links(html)
+ # Check if the document is empty
+ # Prevents edge-case bug with Reddit comments, see issue #3115
+ if html.nil? || html.empty?
+ return html
+ end
+
+ html = XML.parse_html(html)
+
+ html.xpath_nodes(%q(//a)).each do |anchor|
+ url = URI.parse(anchor["href"])
+
+ if url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") || url.host.not_nil!.ends_with?("youtu.be")
+ if url.host.try &.ends_with? "youtu.be"
+ url = "/watch?v=#{url.path.lstrip('/')}#{url.query_params}"
+ else
+ if url.path == "/redirect"
+ params = HTTP::Params.parse(url.query.not_nil!)
+ anchor["href"] = params["q"]?
+ else
+ anchor["href"] = url.request_target
+ end
+ end
+ elsif url.to_s == "#"
+ begin
+ length_seconds = decode_length_seconds(anchor.content)
+ rescue ex
+ length_seconds = decode_time(anchor.content)
+ end
+
+ if length_seconds > 0
+ anchor["href"] = "javascript:void(0)"
+ anchor["onclick"] = "player.currentTime(#{length_seconds})"
+ else
+ anchor["href"] = url.request_target
+ end
+ end
+ end
+
+ html = html.xpath_node(%q(//body)).not_nil!
+ if node = html.xpath_node(%q(./p))
+ html = node
+ end
+
+ return html.to_xml(options: XML::SaveOptions::NO_DECL)
+end
+
+def fill_links(html, scheme, host)
+ # Check if the document is empty
+ # Prevents edge-case bug with Reddit comments, see issue #3115
+ if html.nil? || html.empty?
+ return html
+ end
+
+ html = XML.parse_html(html)
+
+ html.xpath_nodes("//a").each do |match|
+ url = URI.parse(match["href"])
+ # Reddit links don't have host
+ if !url.host && !match["href"].starts_with?("javascript") && !url.to_s.ends_with? "#"
+ url.scheme = scheme
+ url.host = host
+ match["href"] = url
+ end
+ end
+
+ if host == "www.youtube.com"
+ html = html.xpath_node(%q(//body/p)).not_nil!
+ end
+
+ return html.to_xml(options: XML::SaveOptions::NO_DECL)
+end
+
+def text_to_parsed_content(text : String) : JSON::Any
+ nodes = [] of JSON::Any
+ # For each line convert line to array of nodes
+ text.split('\n').each do |line|
+ # In first case line is just a simple node before
+ # check patterns inside line
+ # { 'text': line }
+ currentNodes = [] of JSON::Any
+ initialNode = {"text" => line}
+ currentNodes << (JSON.parse(initialNode.to_json))
+
+ # For each match with url pattern, get last node and preserve
+ # last node before create new node with url information
+ # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } }
+ line.scan(/https?:\/\/[^ ]*/).each do |urlMatch|
+ # Retrieve last node and update node without match
+ lastNode = currentNodes[currentNodes.size - 1].as_h
+ splittedLastNode = lastNode["text"].as_s.split(urlMatch[0])
+ lastNode["text"] = JSON.parse(splittedLastNode[0].to_json)
+ currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json)
+ # Create new node with match and navigation infos
+ currentNode = {"text" => urlMatch[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => urlMatch[0]}}}
+ currentNodes << (JSON.parse(currentNode.to_json))
+ # If text remain after match create new simple node with text after match
+ afterNode = {"text" => splittedLastNode.size > 0 ? splittedLastNode[1] : ""}
+ currentNodes << (JSON.parse(afterNode.to_json))
+ end
+
+ # After processing of matches inside line
+ # Add \n at end of last node for preserve carriage return
+ lastNode = currentNodes[currentNodes.size - 1].as_h
+ lastNode["text"] = JSON.parse("#{currentNodes[currentNodes.size - 1]["text"]}\n".to_json)
+ currentNodes[currentNodes.size - 1] = JSON.parse(lastNode.to_json)
+
+ # Finally add final nodes to nodes returned
+ currentNodes.each do |node|
+ nodes << (node)
+ end
+ end
+ return JSON.parse({"runs" => nodes}.to_json)
+end
+
+def parse_content(content : JSON::Any, video_id : String? = "") : String
+ content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s ||
+ content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r, video_id).try &.to_s.gsub("\n", "
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
+ + [ − ] + #{child.author} + #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} + #{translate(locale, "`x` ago", recode_date(child.created_utc, locale))} + #{translate(locale, "permalink")} +
+
+ #{body_html}
+ #{replies_html}
+
+ ") } || "" +end + +def content_to_comment_html(content, video_id : String? = "") + html_array = content.map do |run| + # Sometimes, there is an empty element. + # See: https://github.com/iv-org/invidious/issues/3096 + next if run.as_h.empty? + + text = HTML.escape(run["text"].as_s) + + if run["navigationEndpoint"]? + if url = run["navigationEndpoint"]["urlEndpoint"]?.try &.["url"].as_s + url = URI.parse(url) + displayed_url = text + + if url.host == "youtu.be" + url = "/watch?v=#{url.request_target.lstrip('/')}" + elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") + if url.path == "/redirect" + # Sometimes, links can be corrupted (why?) so make sure to fallback + # nicely. See https://github.com/iv-org/invidious/issues/2682 + url = url.query_params["q"]? || "" + displayed_url = url + else + url = url.request_target + displayed_url = "youtube.com#{url}" + end + end + + text = %(#{reduce_uri(displayed_url)}) + elsif watch_endpoint = run["navigationEndpoint"]["watchEndpoint"]? + start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i + link_video_id = watch_endpoint["videoId"].as_s + + url = "/watch?v=#{link_video_id}" + url += "&t=#{start_time}" if !start_time.nil? + + # If the current video ID (passed through from the caller function) + # is the same as the video ID in the link, add HTML attributes for + # the JS handler function that bypasses page reload. + # + # See: https://github.com/iv-org/invidious/issues/3063 + if link_video_id == video_id + start_time ||= 0 + text = %(#{reduce_uri(text)}) + else + text = %(#{text}) + end + elsif url = run.dig?("navigationEndpoint", "commandMetadata", "webCommandMetadata", "url").try &.as_s + if text.starts_with?(/\s?[@#]/) + # Handle "pings" in comments and hasthags differently + # See: + # - https://github.com/iv-org/invidious/issues/3038 + # - https://github.com/iv-org/invidious/issues/3062 + text = %(#{text}) + else + text = %(#{reduce_uri(url)}) + end + end + end + + text = "#{text}" if run["bold"]? + text = "#{text}" if run["italics"]? + + text + end + + return html_array.join("").delete('\ufeff') +end + +def produce_comment_continuation(video_id, cursor = "", sort_by = "top") + object = { + "2:embedded" => { + "2:string" => video_id, + "25:varint" => 0_i64, + "28:varint" => 1_i64, + "36:embedded" => { + "5:varint" => -1_i64, + "8:varint" => 0_i64, + }, + "40:embedded" => { + "1:varint" => 4_i64, + "3:string" => "https://www.youtube.com", + "4:string" => "", + }, + }, + "3:varint" => 6_i64, + "6:embedded" => { + "1:string" => cursor, + "4:embedded" => { + "4:string" => video_id, + "6:varint" => 0_i64, + }, + "5:varint" => 20_i64, + }, + } + + case sort_by + when "top" + object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 + when "new", "newest" + object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64 + else # top + object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 + end + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr deleted file mode 100644 index 1f55bfe6..00000000 --- a/src/invidious/comments/content.cr +++ /dev/null @@ -1,89 +0,0 @@ -def text_to_parsed_content(text : String) : JSON::Any - nodes = [] of JSON::Any - # For each line convert line to array of nodes - text.split('\n').each do |line| - # In first case line is just a simple node before - # check patterns inside line - # { 'text': line } - current_nodes = [] of JSON::Any - initial_node = {"text" => line} - current_nodes << (JSON.parse(initial_node.to_json)) - - # For each match with url pattern, get last node and preserve - # last node before create new node with url information - # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } - line.scan(/https?:\/\/[^ ]*/).each do |url_match| - # Retrieve last node and update node without match - last_node = current_nodes[-1].as_h - splitted_last_node = last_node["text"].as_s.split(url_match[0]) - last_node["text"] = JSON.parse(splitted_last_node[0].to_json) - current_nodes[-1] = JSON.parse(last_node.to_json) - # Create new node with match and navigation infos - current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}} - current_nodes << (JSON.parse(current_node.to_json)) - # If text remain after match create new simple node with text after match - after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""} - current_nodes << (JSON.parse(after_node.to_json)) - end - - # After processing of matches inside line - # Add \n at end of last node for preserve carriage return - last_node = current_nodes[-1].as_h - last_node["text"] = JSON.parse("#{last_node["text"]}\n".to_json) - current_nodes[-1] = JSON.parse(last_node.to_json) - - # Finally add final nodes to nodes returned - current_nodes.each do |node| - nodes << (node) - end - end - return JSON.parse({"runs" => nodes}.to_json) -end - -def parse_content(content : JSON::Any, video_id : String? = "") : String - content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || - content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r, video_id).try &.to_s.gsub("\n", "
") } || "" -end - -def content_to_comment_html(content, video_id : String? = "") - html_array = content.map do |run| - # Sometimes, there is an empty element. - # See: https://github.com/iv-org/invidious/issues/3096 - next if run.as_h.empty? - - text = HTML.escape(run["text"].as_s) - - if navigation_endpoint = run.dig?("navigationEndpoint") - text = parse_link_endpoint(navigation_endpoint, text, video_id) - end - - text = "#{text}" if run["bold"]? - text = "
\n)
-
- if tab == selected_tab
- str << "\t"
- str << translate(locale, "channel_tab_#{tab_name}_label")
- str << "\n"
- else
- # Video tab doesn't have the last path component
- url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
-
- str << %(\t)
- str << translate(locale, "channel_tab_#{tab_name}_label")
- str << "\n"
- end
-
- str << "
"
- end
- end
- end
- end
-end
diff --git a/src/invidious/frontend/comments_reddit.cr b/src/invidious/frontend/comments_reddit.cr
deleted file mode 100644
index 4dda683e..00000000
--- a/src/invidious/frontend/comments_reddit.cr
+++ /dev/null
@@ -1,50 +0,0 @@
-module Invidious::Frontend::Comments
- extend self
-
- def template_reddit(root, locale)
- String.build do |html|
- root.each do |child|
- if child.data.is_a?(RedditComment)
- child = child.data.as(RedditComment)
- body_html = HTML.unescape(child.body_html)
-
- replies_html = ""
- if child.replies.is_a?(RedditThing)
- replies = child.replies.as(RedditThing)
- replies_html = self.template_reddit(replies.data.as(RedditListing).children, locale)
- end
-
- if child.depth > 0
- html << <<-END_HTML
-
-
- #{go_to_youtube}
+ #{go_to_youtube}
END_HTML
diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr
index 13ea9fe9..d140a858 100644
--- a/src/invidious/helpers/handlers.cr
+++ b/src/invidious/helpers/handlers.cr
@@ -27,7 +27,6 @@ class Kemal::RouteHandler
# Processes the route if it's a match. Otherwise renders 404.
private def process_request(context)
raise Kemal::Exceptions::RouteNotFound.new(context) unless context.route_found?
- return if context.response.closed?
content = context.route.handler.call(context)
if !Kemal.config.error_handlers.empty? && Kemal.config.error_handlers.has_key?(context.response.status_code) && exclude_match?(context)
@@ -98,7 +97,7 @@ class AuthHandler < Kemal::Handler
if token = env.request.headers["Authorization"]?
token = JSON.parse(URI.decode_www_form(token.lchop("Bearer ")))
session = URI.decode_www_form(token["session"].as_s)
- scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil)
+ scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil)
if email = Invidious::Database::SessionIDs.select_email(session)
user = Invidious::Database::Users.select!(email: email)
@@ -143,8 +142,63 @@ class APIHandler < Kemal::Handler
exclude ["/api/v1/auth/notifications"], "POST"
def call(env)
- env.response.headers["Access-Control-Allow-Origin"] = "*" if only_match?(env)
- call_next env
+ return call_next env unless only_match? env
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ # Since /api/v1/notifications is an event-stream, we don't want
+ # to wrap the response
+ return call_next env if exclude_match? env
+
+ # Here we swap out the socket IO so we can modify the response as needed
+ output = env.response.output
+ env.response.output = IO::Memory.new
+
+ begin
+ call_next env
+
+ env.response.output.rewind
+
+ if env.response.output.as(IO::Memory).size != 0 &&
+ env.response.headers.includes_word?("Content-Type", "application/json")
+ response = JSON.parse(env.response.output)
+
+ if fields_text = env.params.query["fields"]?
+ begin
+ JSONFilter.filter(response, fields_text)
+ rescue ex
+ env.response.status_code = 400
+ response = {"error" => ex.message}
+ end
+ end
+
+ if env.params.query["pretty"]?.try &.== "1"
+ response = response.to_pretty_json
+ else
+ response = response.to_json
+ end
+ else
+ response = env.response.output.gets_to_end
+ end
+ rescue ex
+ env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html")
+ env.response.status_code = 500
+
+ if env.response.headers.includes_word?("Content-Type", "application/json")
+ response = {"error" => ex.message || "Unspecified error"}
+
+ if env.params.query["pretty"]?.try &.== "1"
+ response = response.to_pretty_json
+ else
+ response = response.to_json
+ end
+ end
+ ensure
+ env.response.output = output
+ env.response.print response
+
+ env.response.flush
+ end
end
end
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index 6add0237..c3b53339 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -22,6 +22,31 @@ struct Annotation
property annotations : String
end
+def login_req(f_req)
+ data = {
+ # Unfortunately there's not much information available on `bgRequest`; part of Google's BotGuard
+ # Generally this is much longer (>1250 characters), see also
+ # https://github.com/ytdl-org/youtube-dl/commit/baf67a604d912722b0fe03a40e9dc5349a2208cb .
+ # For now this can be empty.
+ "bgRequest" => %|["identifier",""]|,
+ "pstMsg" => "1",
+ "checkConnection" => "youtube",
+ "checkedDomains" => "youtube",
+ "hl" => "en",
+ "deviceinfo" => %|[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]|,
+ "f.req" => f_req,
+ "flowName" => "GlifWebSignIn",
+ "flowEntry" => "ServiceLogin",
+ # "cookiesDisabled" => "false",
+ # "gmscoreversion" => "undefined",
+ # "continue" => "https://accounts.google.com/ManageAccount",
+ # "azt" => "",
+ # "bgHash" => "",
+ }
+
+ return HTTP::Params.encode(data)
+end
+
def html_to_content(description_html : String)
description = description_html.gsub(/(
)|(
": "\n", @@ -78,6 +103,15 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -104,6 +138,15 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -137,6 +180,15 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -181,20 +233,3 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end - -# Fetch the playback requests tracker from the statistics endpoint. -# -# Creates a new tracker when unavailable. -def get_playback_statistic - if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? - tracker = { - "totalRequests" => 0_i64, - "successfulRequests" => 0_i64, - "ratio" => 0_f64, - } - - Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker - end - - return tracker.as(Hash(String, Int64 | Float64)) -end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index bca2edda..fd86594c 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,22 +1,9 @@ -# Languages requiring a better level of translation (at least 20%) -# to be added to the list below: -# -# "af" => "", # Afrikaans -# "az" => "", # Azerbaijani -# "be" => "", # Belarusian -# "bn_BD" => "", # Bengali (Bangladesh) -# "ia" => "", # Interlingua -# "or" => "", # Odia -# "tk" => "", # Turkmen -# "tok => "", # Toki Pona -# +# "bn_BD" => load_locale("bn_BD"), # Bengali (Bangladesh) [Incomplete] +# "eu" => load_locale("eu"), # Basque [Incomplete] +# "sk" => load_locale("sk"), # Slovak [Incomplete] LOCALES_LIST = { "ar" => "العربية", # Arabic - "bg" => "български", # Bulgarian - "bn" => "বাংলা", # Bengali - "ca" => "Català", # Catalan "cs" => "Čeština", # Czech - "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek @@ -24,7 +11,6 @@ LOCALES_LIST = { "eo" => "Esperanto", # Esperanto "es" => "Español", # Spanish "et" => "Eesti keel", # Estonian - "eu" => "Euskara", # Basque "fa" => "فارسی", # Persian "fi" => "Suomi", # Finnish "fr" => "Français", # French @@ -37,7 +23,6 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean - "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch @@ -47,14 +32,11 @@ LOCALES_LIST = { "pt-PT" => "Português de Portugal", # Portuguese (Portugal) "ro" => "Română", # Romanian "ru" => "Русский", # Russian - "si" => "සිංහල", # Sinhala - "sk" => "Slovenčina", # Slovak "sl" => "Slovenščina", # Slovenian "sq" => "Shqip", # Albanian "sr" => "Srpski (latinica)", # Serbian (Latin) "sr_Cyrl" => "Српски (ћирилица)", # Serbian (Cyrillic) "sv-SE" => "Svenska", # Swedish - "ta" => "தமிழ்", # Tamil "tr" => "Türkçe", # Turkish "uk" => "Українська", # Ukrainian "vi" => "Tiếng Việt", # Vietnamese @@ -94,7 +76,7 @@ def load_all_locales return locales end -def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String +def translate(locale : String?, key : String, text : String | Nil = nil) : String # Log a warning if "key" doesn't exist in en-US locale and return # that key as the text, so this is more or less transparent to the user. if !LOCALES["en-US"].has_key?(key) @@ -117,12 +99,10 @@ def translate(locale : String?, key : String, text : String | Hash(String, Strin match_length = 0 raw_data.as_h.each do |hash_key, value| - if text.is_a?(String) - if md = text.try &.match(/#{hash_key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size - end + if md = text.try &.match(/#{hash_key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size end end end @@ -132,13 +112,8 @@ def translate(locale : String?, key : String, text : String | Hash(String, Strin raise "Invalid translation \"#{raw_data}\"" end - if text.is_a?(String) + if text translation = translation.gsub("`x`", text) - elsif text.is_a?(Hash(String, String)) - # adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic - text.each_key do |hash_key| - translation = translation.gsub("{{#{hash_key}}}", text[hash_key]) - end end return translation @@ -188,12 +163,3 @@ def translate_bool(locale : String?, translation : Bool) return translate(locale, "No") end end - -def locale_is_rtl?(locale : String?) - # Fallback to en-US - return false if locale.nil? - - # Arabic, Persian, Hebrew - # See https://en.wikipedia.org/wiki/Right-to-left_script#List_of_RTL_scripts - return {"ar", "fa", "he"}.includes? locale -end diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index 684e6d14..e84f88fb 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -35,35 +35,27 @@ module I18next::Plurals Special_Slovenian = 21 Special_Hebrew = 22 Special_Odia = 23 - - # Mixed v3/v4 rules in Weblate - # `es`, `pt` and `pt-PT` doesn't seem to have been refreshed - # by weblate yet, but I suspect it will happen one day. - # See: https://github.com/translate/translate/issues/4873 - Special_French_Portuguese - Special_Hungarian_Serbian - Special_Spanish_Italian end private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fil", "fr", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt", "pt-BR", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", - "eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", - "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", + "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "hu", "hy", "ia", "it", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", - "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", + "ps", "pt-PT", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ - "ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ - "be", "bs", "cnr", "dz", "ru", "uk", + "be", "bs", "cnr", "dz", "hr", "ru", "sr", "uk", ], } @@ -89,13 +81,6 @@ module I18next::Plurals "ro" => PluralForms::Special_Romanian, "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, - # Mixed v3/v4 rules - "es" => PluralForms::Special_Spanish_Italian, - "fr" => PluralForms::Special_French_Portuguese, - "hr" => PluralForms::Special_Hungarian_Serbian, - "it" => PluralForms::Special_Spanish_Italian, - "pt" => PluralForms::Special_French_Portuguese, - "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -165,8 +150,9 @@ module I18next::Plurals end def get_plural_form(locale : String) : PluralForms - # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code - if !locale.matches?(/^pt-PT$/) + # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code, + # except for pt-BR and pt-PT which needs to be kept as-is. + if !locale.matches?(/^pt-(BR|PT)$/) locale = locale.split('-')[0] end @@ -188,7 +174,7 @@ module I18next::Plurals # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check # from original i18next code - private def simple_plural?(form : PluralForms) : Bool + private def is_simple_plural(form : PluralForms) : Bool case form when .single_gt_one? then return true when .single_not_one? then return true @@ -210,7 +196,7 @@ module I18next::Plurals idx = SuffixIndex.get_index(plural_form, count) # Simple plurals are handled differently in all versions (but v4) - if @simplify_plural_suffix && simple_plural?(plural_form) + if @simplify_plural_suffix && is_simple_plural(plural_form) return (idx == 1) ? "_plural" : "" end @@ -260,10 +246,6 @@ module I18next::Plurals when .special_slovenian? then return special_slovenian(count) when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) - # Mixed v3/v4 forms - when .special_spanish_italian? then return special_cldr_spanish_italian(count) - when .special_french_portuguese? then return special_cldr_french_portuguese(count) - when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count) else # default, if nothing matched above return 0_u8 @@ -525,42 +507,5 @@ module I18next::Plurals def self.special_odia(count : Int) : UInt8 return (count == 1) ? 0_u8 : 1_u8 end - - # ------------------- - # "v3.5" rules - # ------------------- - - # Plural form for Spanish & Italian languages - # - # This rule is mostly compliant to CLDR v42 - # - def self.special_cldr_spanish_italian(count : Int) : UInt8 - return 0_u8 if (count == 1) # one - return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many - return 2_u8 # other - end - - # Plural form for French and Portuguese - # - # This rule is mostly compliant to CLDR v42 - # - def self.special_cldr_french_portuguese(count : Int) : UInt8 - return 0_u8 if (count == 0 || count == 1) # one - return 1_u8 if (count % 1_000_000 == 0) # many - return 2_u8 # other - end - - # Plural form for Hungarian and Serbian - # - # This rule is mostly compliant to CLDR v42 - # - def self.special_cldr_hungarian_serbian(count : Int) : UInt8 - n_mod_10 = count % 10 - n_mod_100 = count % 100 - - return 0_u8 if (n_mod_10 == 1 && n_mod_100 != 11) # one - return 1_u8 if (2 <= n_mod_10 <= 4 && (n_mod_100 < 12 || 14 < n_mod_100)) # few - return 2_u8 # other - end end end diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr new file mode 100644 index 00000000..b8e8f96d --- /dev/null +++ b/src/invidious/helpers/json_filter.cr @@ -0,0 +1,248 @@ +module JSONFilter + alias BracketIndex = Hash(Int64, Int64) + + alias GroupedFieldsValue = String | Array(GroupedFieldsValue) + alias GroupedFieldsList = Array(GroupedFieldsValue) + + class FieldsParser + class ParseError < Exception + end + + # Returns the `Regex` pattern used to match nest groups + def self.nest_group_pattern : Regex + # uses a '.' character to match json keys as they are allowed + # to contain any unicode codepoint + /(?:|,)(?[^,\n]*?)\(/
+ end
+
+ # Returns the `Regex` pattern used to check if there are any empty nest groups
+ def self.unnamed_nest_group_pattern : Regex
+ /^\(|\(\(|\/\(/
+ end
+
+ def self.parse_fields(fields_text : String) : Nil
+ if fields_text.empty?
+ raise FieldsParser::ParseError.new "Fields is empty"
+ end
+
+ opening_bracket_count = fields_text.count('(')
+ closing_bracket_count = fields_text.count(')')
+
+ if opening_bracket_count != closing_bracket_count
+ bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing"
+ raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})"
+ elsif match_result = unnamed_nest_group_pattern.match(fields_text)
+ raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}"
+ end
+
+ # first, handle top-level single nested properties: items/id, playlistItems/snippet, etc
+ parse_single_nests(fields_text) { |nest_list| yield nest_list }
+
+ # next, handle nest groups: items(id, etag, etc)
+ parse_nest_groups(fields_text) { |nest_list| yield nest_list }
+ end
+
+ def self.parse_single_nests(fields_text : String) : Nil
+ single_nests = remove_nest_groups(fields_text)
+
+ if !single_nests.empty?
+ property_nests = single_nests.split(',')
+
+ property_nests.each do |nest|
+ nest_list = nest.split('/')
+ if nest_list.includes? ""
+ raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}"
+ end
+ yield nest_list
+ end
+ # else
+ # raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}"
+ end
+ end
+
+ def self.parse_nest_groups(fields_text : String) : Nil
+ nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64)
+ bracket_pairs = get_bracket_pairs(fields_text, true)
+
+ text_index = 0
+ regex_index = 0
+
+ while regex_result = self.nest_group_pattern.match(fields_text, regex_index)
+ raw_match = regex_result[0]
+ group_name = regex_result["groupname"]
+
+ text_index = regex_result.begin
+ regex_index = regex_result.end
+
+ if text_index.nil? || regex_index.nil?
+ raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}"
+ end
+
+ offset = raw_match.starts_with?(',') ? 1 : 0
+
+ opening_bracket_index = (text_index + group_name.size) + offset
+ closing_bracket_index = bracket_pairs[opening_bracket_index]
+ content_start = opening_bracket_index + 1
+
+ content = fields_text[content_start...closing_bracket_index]
+
+ if content.empty?
+ raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}"
+ else
+ content = remove_nest_groups(content)
+ end
+
+ while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index]
+ if nest_stack.size
+ nest_stack.pop
+ end
+ end
+
+ group_name.split('/').each do |name|
+ nest_stack.push({
+ group_name: name,
+ closing_bracket_index: closing_bracket_index,
+ })
+ end
+
+ if !content.empty?
+ properties = content.split(',')
+
+ properties.each do |prop|
+ nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] }
+
+ if !prop.empty?
+ if prop.includes?('/')
+ parse_single_nests(prop) { |list| nest_list += list }
+ else
+ nest_list.push prop
+ end
+ else
+ raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}"
+ end
+
+ yield nest_list
+ end
+ end
+ end
+ end
+
+ def self.remove_nest_groups(text : String) : String
+ content_bracket_pairs = get_bracket_pairs(text, false)
+
+ content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket|
+ closing_bracket = content_bracket_pairs[opening_bracket]
+ last_comma = text.rindex(',', opening_bracket) || 0
+
+ text = text[0...last_comma] + text[closing_bracket + 1...text.size]
+ end
+
+ return text.starts_with?(',') ? text[1...text.size] : text
+ end
+
+ def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex
+ istart = [] of Int64
+ bracket_index = BracketIndex.new
+
+ text.each_char_with_index do |char, index|
+ if char == '('
+ istart.push(index.to_i64)
+ end
+
+ if char == ')'
+ begin
+ opening = istart.pop
+ if recursive || (!recursive && istart.size == 0)
+ bracket_index[opening] = index.to_i64
+ end
+ rescue
+ raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}"
+ end
+ end
+ end
+
+ if istart.size != 0
+ idx = istart.pop
+ raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}"
+ end
+
+ return bracket_index
+ end
+ end
+
+ class FieldsGrouper
+ alias SkeletonValue = Hash(String, SkeletonValue)
+
+ def self.create_json_skeleton(fields_text : String) : SkeletonValue
+ root_hash = {} of String => SkeletonValue
+
+ FieldsParser.parse_fields(fields_text) do |nest_list|
+ current_item = root_hash
+ nest_list.each do |key|
+ if current_item[key]?
+ current_item = current_item[key]
+ else
+ current_item[key] = {} of String => SkeletonValue
+ current_item = current_item[key]
+ end
+ end
+ end
+ root_hash
+ end
+
+ def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList
+ grouped_fields_list = GroupedFieldsList.new
+ json_skeleton.each do |key, value|
+ grouped_fields_list.push key
+
+ nested_keys = create_grouped_fields_list(value)
+ grouped_fields_list.push nested_keys unless nested_keys.empty?
+ end
+ return grouped_fields_list
+ end
+ end
+
+ class FilterError < Exception
+ end
+
+ def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true)
+ skeleton = FieldsGrouper.create_json_skeleton(fields_text)
+ grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton)
+ filter(item, grouped_fields_list, in_place)
+ end
+
+ def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any
+ item = item.clone unless in_place
+
+ if !item.as_h? && !item.as_a?
+ raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}"
+ end
+
+ top_level_keys = Array(String).new
+ grouped_fields_list.each do |value|
+ if value.is_a? String
+ top_level_keys.push value
+ elsif value.is_a? Array
+ if !top_level_keys.empty?
+ key_to_filter = top_level_keys.last
+
+ if item.as_h?
+ filter(item[key_to_filter], value, in_place: true)
+ elsif item.as_a?
+ item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) }
+ end
+ else
+ raise FilterError.new "Tried to filter while top level keys list is empty"
+ end
+ end
+ end
+
+ if item.as_h?
+ item.as_h.select! top_level_keys
+ elsif item.as_a?
+ item.as_a.map { |value| filter(value, top_level_keys, in_place: true) }
+ end
+
+ item
+ end
+end
diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr
index 03349595..e2e50905 100644
--- a/src/invidious/helpers/logger.cr
+++ b/src/invidious/helpers/logger.cr
@@ -1,5 +1,3 @@
-require "colorize"
-
enum LogLevel
All = 0
Trace = 1
@@ -12,9 +10,7 @@ enum LogLevel
end
class Invidious::LogHandler < Kemal::BaseLogHandler
- def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true)
- Colorize.enabled = use_color
- Colorize.on_tty_only!
+ def initialize(@io : IO = STDOUT, @level = LogLevel::Debug)
end
def call(context : HTTP::Server::Context)
@@ -38,27 +34,28 @@ class Invidious::LogHandler < Kemal::BaseLogHandler
context
end
+ def puts(message : String)
+ @io << message << '\n'
+ @io.flush
+ end
+
def write(message : String)
@io << message
@io.flush
end
- def color(level)
- case level
- when LogLevel::Trace then :cyan
- when LogLevel::Debug then :green
- when LogLevel::Info then :white
- when LogLevel::Warn then :yellow
- when LogLevel::Error then :red
- when LogLevel::Fatal then :magenta
- else :default
- end
+ def set_log_level(level : String)
+ @level = LogLevel.parse(level)
+ end
+
+ def set_log_level(level : LogLevel)
+ @level = level
end
{% for level in %w(trace debug info warn error fatal) %}
def {{level.id}}(message : String)
if LogLevel::{{level.id.capitalize}} >= @level
- puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})))
+ puts("#{Time.utc} [{{level.id}}] #{message}")
end
end
{% end %}
diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr
index 84847321..43e7171b 100644
--- a/src/invidious/helpers/macros.cr
+++ b/src/invidious/helpers/macros.cr
@@ -55,11 +55,12 @@ macro templated(_filename, template = "template", navbar_search = true)
{{ layout = "src/invidious/views/" + template + ".ecr" }}
__content_filename__ = {{filename}}
- render {{filename}}, {{layout}}
+ content = Kilt.render({{filename}})
+ Kilt.render({{layout}})
end
macro rendered(filename)
- render("src/invidious/views/#{{{filename}}}.ecr")
+ Kilt.render("src/invidious/views/#{{{filename}}}.ecr")
end
# Similar to Kemals halt method but works in a
diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr
index 2796a8dc..3918bd13 100644
--- a/src/invidious/helpers/serialized_yt_data.cr
+++ b/src/invidious/helpers/serialized_yt_data.cr
@@ -1,16 +1,3 @@
-@[Flags]
-enum VideoBadges
- LiveNow
- Premium
- ThreeD
- FourK
- New
- EightK
- VR180
- VR360
- ClosedCaptions
-end
-
struct SearchVideo
include DB::Serializable
@@ -22,10 +9,10 @@ struct SearchVideo
property views : Int64
property description_html : String
property length_seconds : Int32
+ property live_now : Bool
+ property premium : Bool
property premiere_timestamp : Time?
property author_verified : Bool
- property author_thumbnail : String?
- property badges : VideoBadges
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
@@ -87,52 +74,25 @@ struct SearchVideo
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
- json.field "authorVerified", self.author_verified
-
- author_thumbnail = self.author_thumbnail
-
- if author_thumbnail
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
- end
json.field "videoThumbnails" do
- Invidious::JSONify::APIv1.thumbnails(json, self.id)
+ generate_thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views
- json.field "viewCountText", translate_count(locale, "generic_views_count", self.views, NumberFormatting::Short)
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds
- json.field "liveNow", self.badges.live_now?
- json.field "premium", self.badges.premium?
- json.field "isUpcoming", self.upcoming?
+ json.field "liveNow", self.live_now
+ json.field "premium", self.premium
+ json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
- json.field "isNew", self.badges.new?
- json.field "is4k", self.badges.four_k?
- json.field "is8k", self.badges.eight_k?
- json.field "isVr180", self.badges.vr180?
- json.field "isVr360", self.badges.vr360?
- json.field "is3d", self.badges.three_d?
- json.field "hasCaptions", self.badges.closed_captions?
end
end
@@ -147,7 +107,7 @@ struct SearchVideo
to_json(nil, json)
end
- def upcoming?
+ def is_upcoming
premiere_timestamp ? true : false
end
end
@@ -195,7 +155,7 @@ struct SearchPlaylist
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
- Invidious::JSONify::APIv1.thumbnails(json, video.id)
+ generate_thumbnails(json, video.id)
end
end
end
@@ -224,7 +184,6 @@ struct SearchChannel
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
- property channel_handle : String?
property description_html : String
property auto_generated : Bool
property author_verified : Bool
@@ -242,7 +201,7 @@ struct SearchChannel
qualities.each do |quality|
json.object do
- json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
+ json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
@@ -253,7 +212,6 @@ struct SearchChannel
json.field "autoGenerated", self.auto_generated
json.field "subCount", self.subscriber_count
json.field "videoCount", self.video_count
- json.field "channelHandle", self.channel_handle
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
@@ -272,74 +230,6 @@ struct SearchChannel
end
end
-struct SearchHashtag
- include DB::Serializable
-
- property title : String
- property url : String
- property video_count : Int64
- property channel_count : Int64
-
- def to_json(locale : String?, json : JSON::Builder)
- json.object do
- json.field "type", "hashtag"
- json.field "title", self.title
- json.field "url", self.url
- json.field "videoCount", self.video_count
- json.field "channelCount", self.channel_count
- end
- end
-end
-
-# A `ProblematicTimelineItem` is a `SearchItem` created by Invidious that
-# represents an item that caused an exception during parsing.
-#
-# This is not a parsed object from YouTube but rather an Invidious-only type
-# created to gracefully communicate parse errors without throwing away
-# the rest of the (hopefully) successfully parsed item on a page.
-struct ProblematicTimelineItem
- property parse_exception : Exception
- property id : String
-
- def initialize(@parse_exception)
- @id = Random.new.hex(8)
- end
-
- def to_json(locale : String?, json : JSON::Builder)
- json.object do
- json.field "type", "parse-error"
- json.field "errorMessage", @parse_exception.message
- json.field "errorBacktrace", @parse_exception.inspect_with_backtrace
- end
- end
-
- # Provides compatibility with PlaylistVideo
- def to_json(json : JSON::Builder, *args, **kwargs)
- return to_json("", json)
- end
-
- def to_xml(env, locale, xml : XML::Builder)
- xml.element("entry") do
- xml.element("id") { xml.text "iv-err-#{@id}" }
- xml.element("title") { xml.text "Parse Error: This item has failed to parse" }
- xml.element("updated") { xml.text Time.utc.to_rfc3339 }
-
- xml.element("content", type: "xhtml") do
- xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
- xml.element("div") do
- xml.element("h4") { translate(locale, "timeline_parse_error_placeholder_heading") }
- xml.element("p") { translate(locale, "timeline_parse_error_placeholder_message") }
- end
-
- xml.element("pre") do
- get_issue_template(env, @parse_exception)
- end
- end
- end
- end
- end
-end
-
class Category
include DB::Serializable
@@ -375,11 +265,4 @@ class Category
end
end
-struct Continuation
- getter token
-
- def initialize(@token : String)
- end
-end
-
-alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category | ProblematicTimelineItem
+alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category
diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr
deleted file mode 100644
index 6d198a42..00000000
--- a/src/invidious/helpers/sig_helper.cr
+++ /dev/null
@@ -1,349 +0,0 @@
-require "uri"
-require "socket"
-require "socket/tcp_socket"
-require "socket/unix_socket"
-
-{% if flag?(:advanced_debug) %}
- require "io/hexdump"
-{% end %}
-
-private alias NetworkEndian = IO::ByteFormat::NetworkEndian
-
-module Invidious::SigHelper
- enum UpdateStatus
- Updated
- UpdateNotRequired
- Error
- end
-
- # -------------------
- # Payload types
- # -------------------
-
- abstract struct Payload
- end
-
- struct StringPayload < Payload
- getter string : String
-
- def initialize(str : String)
- raise Exception.new("SigHelper: String can't be empty") if str.empty?
- @string = str
- end
-
- def self.from_bytes(slice : Bytes)
- size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice)
- if size == 0 # Error code
- raise Exception.new("SigHelper: Server encountered an error")
- end
-
- if (slice.bytesize - 2) != size
- raise Exception.new("SigHelper: String size mismatch")
- end
-
- if str = String.new(slice[2..])
- return self.new(str)
- else
- raise Exception.new("SigHelper: Can't read string from socket")
- end
- end
-
- def to_io(io)
- # `.to_u16` raises if there is an overflow during the conversion
- io.write_bytes(@string.bytesize.to_u16, NetworkEndian)
- io.write(@string.to_slice)
- end
- end
-
- private enum Opcode
- FORCE_UPDATE = 0
- DECRYPT_N_SIGNATURE = 1
- DECRYPT_SIGNATURE = 2
- GET_SIGNATURE_TIMESTAMP = 3
- GET_PLAYER_STATUS = 4
- PLAYER_UPDATE_TIMESTAMP = 5
- end
-
- private record Request,
- opcode : Opcode,
- payload : Payload?
-
- # ----------------------
- # High-level functions
- # ----------------------
-
- class Client
- @mux : Multiplexor
-
- def initialize(uri_or_path)
- @mux = Multiplexor.new(uri_or_path)
- end
-
- # Forces the server to re-fetch the YouTube player, and extract the necessary
- # components from it (nsig function code, sig function code, signature timestamp).
- def force_update : UpdateStatus
- request = Request.new(Opcode::FORCE_UPDATE, nil)
-
- value = send_request(request) do |bytes|
- IO::ByteFormat::NetworkEndian.decode(UInt16, bytes)
- end
-
- case value
- when 0x0000 then return UpdateStatus::Error
- when 0xFFFF then return UpdateStatus::UpdateNotRequired
- when 0xF44F then return UpdateStatus::Updated
- else
- code = value.nil? ? "nil" : value.to_s(base: 16)
- raise Exception.new("SigHelper: Invalid status code received #{code}")
- end
- end
-
- # Decrypt a provided n signature using the server's current nsig function
- # code, and return the result (or an error).
- def decrypt_n_param(n : String) : String?
- request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n))
-
- n_dec = self.send_request(request) do |bytes|
- StringPayload.from_bytes(bytes).string
- end
-
- return n_dec
- end
-
- # Decrypt a provided s signature using the server's current sig function
- # code, and return the result (or an error).
- def decrypt_sig(sig : String) : String?
- request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig))
-
- sig_dec = self.send_request(request) do |bytes|
- StringPayload.from_bytes(bytes).string
- end
-
- return sig_dec
- end
-
- # Return the signature timestamp from the server's current player
- def get_signature_timestamp : UInt64?
- request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil)
-
- return self.send_request(request) do |bytes|
- IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)
- end
- end
-
- # Return the current player's version
- def get_player : UInt32?
- request = Request.new(Opcode::GET_PLAYER_STATUS, nil)
-
- return self.send_request(request) do |bytes|
- has_player = (bytes[0] == 0xFF)
- player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4])
- has_player ? player_version : nil
- end
- end
-
- # Return when the player was last updated
- def get_player_timestamp : UInt64?
- request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil)
-
- return self.send_request(request) do |bytes|
- IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)
- end
- end
-
- private def send_request(request : Request, &)
- channel = @mux.send(request)
- slice = channel.receive
- return yield slice
- rescue ex
- LOGGER.debug("SigHelper: Error when sending a request")
- LOGGER.trace(ex.inspect_with_backtrace)
- return nil
- end
- end
-
- # ---------------------
- # Low level functions
- # ---------------------
-
- class Multiplexor
- alias TransactionID = UInt32
- record Transaction, channel = ::Channel(Bytes).new
-
- @prng = Random.new
- @mutex = Mutex.new
- @queue = {} of TransactionID => Transaction
-
- @conn : Connection
- @uri_or_path : String
-
- def initialize(@uri_or_path)
- @conn = Connection.new(uri_or_path)
- listen
- end
-
- def listen : Nil
- raise "Socket is closed" if @conn.closed?
-
- LOGGER.debug("SigHelper: Multiplexor listening")
-
- spawn do
- loop do
- begin
- receive_data
- rescue ex
- LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...")
- # We close the socket because for some reason is not closed.
- @conn.close
- loop do
- begin
- @conn = Connection.new(@uri_or_path)
- LOGGER.info("SigHelper: Reconnected to SigHelper!")
- rescue ex
- LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")
- sleep 500.milliseconds
- next
- end
- break if !@conn.closed?
- end
- end
- Fiber.yield
- end
- end
- end
-
- def send(request : Request)
- transaction = Transaction.new
- transaction_id = @prng.rand(TransactionID)
-
- # Add transaction to queue
- @mutex.synchronize do
- # On a 32-bits random integer, this should never happen. Though, just in case, ...
- if @queue[transaction_id]?
- raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!")
- end
-
- @queue[transaction_id] = transaction
- end
-
- write_packet(transaction_id, request)
-
- return transaction.channel
- end
-
- def receive_data
- transaction_id, slice = read_packet
-
- @mutex.synchronize do
- if transaction = @queue.delete(transaction_id)
- # Remove transaction from queue and send data to the channel
- transaction.channel.send(slice)
- LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel")
- else
- raise Exception.new("SigHelper: Received transaction was not in queue")
- end
- end
- end
-
- # Read a single packet from the socket
- private def read_packet : {TransactionID, Bytes}
- # Header
- transaction_id = @conn.read_bytes(UInt32, NetworkEndian)
- length = @conn.read_bytes(UInt32, NetworkEndian)
-
- LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}")
-
- if length > 67_000
- raise Exception.new("SigHelper: Packet longer than expected (#{length})")
- end
-
- # Payload
- slice = Bytes.new(length)
- @conn.read(slice) if length > 0
-
- LOGGER.trace("SigHelper: payload = #{slice}")
- LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done")
-
- return transaction_id, slice
- end
-
- # Write a single packet to the socket
- private def write_packet(transaction_id : TransactionID, request : Request)
- LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}")
-
- io = IO::Memory.new(1024)
- io.write_bytes(request.opcode.to_u8, NetworkEndian)
- io.write_bytes(transaction_id, NetworkEndian)
-
- if payload = request.payload
- payload.to_io(io)
- end
-
- @conn.send(io)
- @conn.flush
-
- LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done")
- end
- end
-
- class Connection
- @socket : UNIXSocket | TCPSocket
-
- {% if flag?(:advanced_debug) %}
- @io : IO::Hexdump
- {% end %}
-
- def initialize(host_or_path : String)
- case host_or_path
- when .starts_with?('/')
- # Make sure that the file exists
- if File.exists?(host_or_path)
- @socket = UNIXSocket.new(host_or_path)
- else
- raise Exception.new("SigHelper: '#{host_or_path}' no such file")
- end
- when .starts_with?("tcp://")
- uri = URI.parse(host_or_path)
- @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)
- else
- uri = URI.parse("tcp://#{host_or_path}")
- @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)
- end
- LOGGER.info("SigHelper: Using helper at '#{host_or_path}'")
-
- {% if flag?(:advanced_debug) %}
- @io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true)
- {% end %}
-
- @socket.sync = false
- @socket.blocking = false
- end
-
- def closed? : Bool
- return @socket.closed?
- end
-
- def close : Nil
- @socket.close if !@socket.closed?
- end
-
- def flush(*args, **options)
- @socket.flush(*args, **options)
- end
-
- def send(*args, **options)
- @socket.send(*args, **options)
- end
-
- # Wrap IO functions, with added debug tooling if needed
- {% for function in %w(read read_bytes write write_bytes) %}
- def {{function.id}}(*args, **options)
- {% if flag?(:advanced_debug) %}
- @io.{{function.id}}(*args, **options)
- {% else %}
- @socket.{{function.id}}(*args, **options)
- {% end %}
- end
- {% end %}
- end
-end
diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr
index 82a28fc0..ee09415b 100644
--- a/src/invidious/helpers/signatures.cr
+++ b/src/invidious/helpers/signatures.cr
@@ -1,53 +1,73 @@
-require "http/params"
-require "./sig_helper"
+alias SigProc = Proc(Array(String), Int32, Array(String))
-class Invidious::DecryptFunction
- @last_update : Time = Time.utc - 42.days
+struct DecryptFunction
+ @decrypt_function = [] of {SigProc, Int32}
+ @decrypt_time = Time.monotonic
- def initialize(uri_or_path)
- @client = SigHelper::Client.new(uri_or_path)
- self.check_update
+ def initialize(@use_polling = true)
end
- def check_update
- # If we have updated in the last 5 minutes, do nothing
- return if (Time.utc - @last_update) < 5.minutes
+ def update_decrypt_function
+ @decrypt_function = fetch_decrypt_function
+ end
- # Get the amount of time elapsed since when the player was updated, in the
- # event where multiple invidious processes are run in parallel.
- update_time_elapsed = (@client.get_player_timestamp || 301).seconds
+ private def fetch_decrypt_function(id = "CvFH_6DNRCY")
+ document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
+ url = document.match(/src="(?\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
+ player = YT_POOL.client &.get(url).body
- if update_time_elapsed > 5.minutes
- LOGGER.debug("Signature: Player might be outdated, updating")
- @client.force_update
- @last_update = Time.utc
+ function_name = player.match(/^(?[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
+ function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?[^}]+)}/m).not_nil!["body"]
+ function_body = function_body.split(";")[1..-2]
+
+ var_name = function_body[0][0, 2]
+ var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?(.*?))};/).not_nil!["body"]
+
+ operations = {} of String => SigProc
+ var_body.split("},").each do |operation|
+ op_name = operation.match(/^[^:]+/).not_nil![0]
+ op_body = operation.match(/\{[^}]+/).not_nil![0]
+
+ case op_body
+ when "{a.reverse()"
+ operations[op_name] = ->(a : Array(String), _b : Int32) { a.reverse }
+ when "{a.splice(0,b)"
+ operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
+ else
+ operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
+ end
end
+
+ decrypt_function = [] of {SigProc, Int32}
+ function_body.each do |function|
+ function = function.lchop(var_name).delete("[].")
+
+ op_name = function.match(/[^\(]+/).not_nil![0]
+ value = function.match(/\(\w,(?[\d]+)\)/).not_nil!["value"].to_i
+
+ decrypt_function << {operations[op_name], value}
+ end
+
+ return decrypt_function
end
- def decrypt_nsig(n : String) : String?
- self.check_update
- return @client.decrypt_n_param(n)
- rescue ex
- LOGGER.debug(ex.message || "Signature: Unknown error")
- LOGGER.trace(ex.inspect_with_backtrace)
- return nil
- end
+ def decrypt_signature(fmt : Hash(String, JSON::Any))
+ return "" if !fmt["s"]? || !fmt["sp"]?
- def decrypt_signature(str : String) : String?
- self.check_update
- return @client.decrypt_sig(str)
- rescue ex
- LOGGER.debug(ex.message || "Signature: Unknown error")
- LOGGER.trace(ex.inspect_with_backtrace)
- return nil
- end
+ sp = fmt["sp"].as_s
+ sig = fmt["s"].as_s.split("")
+ if !@use_polling
+ now = Time.monotonic
+ if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0
+ @decrypt_function = fetch_decrypt_function
+ @decrypt_time = Time.monotonic
+ end
+ end
- def get_sts : UInt64?
- self.check_update
- return @client.get_signature_timestamp
- rescue ex
- LOGGER.debug(ex.message || "Signature: Unknown error")
- LOGGER.trace(ex.inspect_with_backtrace)
- return nil
+ @decrypt_function.each do |proc, value|
+ sig = proc.call(sig, value)
+ end
+
+ return "&#{sp}=#{sig.join("")}"
end
end
diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr
index 5637e533..8ae5034a 100644
--- a/src/invidious/helpers/utils.cr
+++ b/src/invidious/helpers/utils.cr
@@ -52,9 +52,9 @@ def recode_length_seconds(time)
end
def decode_interval(string : String) : Time::Span
- raw_minutes = string.try &.to_i32?
+ rawMinutes = string.try &.to_i32?
- if !raw_minutes
+ if !rawMinutes
hours = /(?\d+)h/.match(string).try &.["hours"].try &.to_i32
hours ||= 0
@@ -63,7 +63,7 @@ def decode_interval(string : String) : Time::Span
time = Time::Span.new(hours: hours, minutes: minutes)
else
- time = Time::Span.new(minutes: raw_minutes)
+ time = Time::Span.new(minutes: rawMinutes)
end
return time
@@ -111,27 +111,24 @@ def decode_date(string : String)
else nil # Continue
end
- # String matches format "20 hours ago", "4 months ago", "20s ago", "15min ago"...
- match = string.match(/(?\d+) ?(?[smhdwy]\w*) ago/)
+ # String matches format "20 hours ago", "4 months ago"...
+ date = string.split(" ")[-3, 3]
+ delta = date[0].to_i
- raise "Could not parse #{string}" if match.nil?
-
- delta = match["count"].to_i
-
- case match["span"]
- when .starts_with? "s" # second(s)
+ case date[1]
+ when .includes? "second"
delta = delta.seconds
- when .starts_with? "mi" # minute(s)
+ when .includes? "minute"
delta = delta.minutes
- when .starts_with? "h" # hour(s)
+ when .includes? "hour"
delta = delta.hours
- when .starts_with? "d" # day(s)
+ when .includes? "day"
delta = delta.days
- when .starts_with? "w" # week(s)
+ when .includes? "week"
delta = delta.weeks
- when .starts_with? "mo" # month(s)
+ when .includes? "month"
delta = delta.months
- when .starts_with? "y" # year(s)
+ when .includes? "year"
delta = delta.years
else
raise "Could not parse #{string}"
@@ -164,19 +161,21 @@ def number_with_separator(number)
number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse
end
-def short_text_to_number(short_text : String) : Int64
- matches = /(?\d+(\.\d+)?)\s?(?[mMkKbB]?)/.match(short_text)
- number = matches.try &.["number"].to_f || 0.0
-
- case matches.try &.["suffix"].downcase
- when "k" then number *= 1_000
- when "m" then number *= 1_000_000
- when "b" then number *= 1_000_000_000
+def short_text_to_number(short_text : String) : Int32
+ case short_text
+ when .ends_with? "M"
+ number = short_text.rstrip(" mM").to_f
+ number *= 1000000
+ when .ends_with? "K"
+ number = short_text.rstrip(" kK").to_f
+ number *= 1000
+ else
+ number = short_text.rstrip(" ")
end
- return number.to_i64
-rescue ex
- return 0_i64
+ number = number.to_i
+
+ return number
end
def number_to_short_text(number)
@@ -262,7 +261,7 @@ def get_referer(env, fallback = "/", unroll = true)
end
referer = referer.request_target
- referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z+]/, "").lstrip("/\\")
+ referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\")
if referer == env.request.path
referer = fallback
@@ -323,6 +322,68 @@ def parse_range(range)
return 0_i64, nil
end
+def fetch_random_instance
+ begin
+ instance_api_client = make_client(URI.parse("https://api.invidious.io"))
+
+ # Timeouts
+ instance_api_client.connect_timeout = 10.seconds
+ instance_api_client.dns_timeout = 10.seconds
+
+ instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a
+ instance_api_client.close
+ rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException
+ instance_list = [] of JSON::Any
+ end
+
+ filtered_instance_list = [] of String
+
+ instance_list.each do |data|
+ # TODO Check if current URL is onion instance and use .onion types if so.
+ if data[1]["type"] == "https"
+ # Instances can have statistics disabled, which is an requirement of version validation.
+ # as_nil? doesn't exist. Thus we'll have to handle the error raised if as_nil fails.
+ begin
+ data[1]["stats"].as_nil
+ next
+ rescue TypeCastError
+ end
+
+ # stats endpoint could also lack the software dict.
+ next if data[1]["stats"]["software"]?.nil?
+
+ # Makes sure the instance isn't too outdated.
+ if remote_version = data[1]["stats"]?.try &.["software"]?.try &.["version"]
+ remote_commit_date = remote_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/)
+ next if !remote_commit_date
+
+ remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC)
+ local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC)
+
+ next if (remote_commit_date - local_commit_date).abs.days > 30
+
+ begin
+ data[1]["monitor"].as_nil
+ health = data[1]["monitor"].as_h["dailyRatios"][0].as_h["ratio"]
+ filtered_instance_list << data[0].as_s if health.to_s.to_f > 90
+ rescue TypeCastError
+ # We can't check the health if the monitoring is broken. Thus we'll just add it to the list
+ # and move on. Ideally we'll ignore any instance that has broken health monitoring but due to the fact that
+ # it's an error that often occurs with all the instances at the same time, we have to just skip the check.
+ filtered_instance_list << data[0].as_s
+ end
+ end
+ end
+ end
+
+ # If for some reason no instances managed to get fetched successfully then we'll just redirect to redirect.invidious.io
+ if filtered_instance_list.size == 0
+ return "redirect.invidious.io"
+ end
+
+ return filtered_instance_list.sample(1)[0]
+end
+
def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "…") : String
str = uri.to_s.sub(/^https?:\/\//, "")
if str.size > max_length
@@ -330,75 +391,3 @@ def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "
end
return str
end
-
-# Get the html link from a NavigationEndpoint or an innertubeCommand
-def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
- if url = endpoint.dig?("urlEndpoint", "url").try &.as_s
- url = URI.parse(url)
- displayed_url = text
-
- if url.host == "youtu.be"
- url = "/watch?v=#{url.request_target.lstrip('/')}"
- elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com")
- if url.path == "/redirect"
- # Sometimes, links can be corrupted (why?) so make sure to fallback
- # nicely. See https://github.com/iv-org/invidious/issues/2682
- url = url.query_params["q"]? || ""
- displayed_url = url
- else
- url = url.request_target
- displayed_url = "youtube.com#{url}"
- end
- end
-
- text = %(#{reduce_uri(displayed_url)})
- elsif watch_endpoint = endpoint.dig?("watchEndpoint")
- start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i
- link_video_id = watch_endpoint["videoId"].as_s
-
- url = "/watch?v=#{link_video_id}"
- url += "&t=#{start_time}" if !start_time.nil?
-
- # If the current video ID (passed through from the caller function)
- # is the same as the video ID in the link, add HTML attributes for
- # the JS handler function that bypasses page reload.
- #
- # See: https://github.com/iv-org/invidious/issues/3063
- if link_video_id == video_id
- start_time ||= 0
- text = %(#{reduce_uri(text)})
- else
- text = %(#{text})
- end
- elsif url = endpoint.dig?("commandMetadata", "webCommandMetadata", "url").try &.as_s
- if text.starts_with?(/\s?[@#]/)
- # Handle "pings" in comments and hasthags differently
- # See:
- # - https://github.com/iv-org/invidious/issues/3038
- # - https://github.com/iv-org/invidious/issues/3062
- text = %(#{text})
- else
- text = %(#{reduce_uri(text)})
- end
- end
- return text
-end
-
-def encrypt_ecb_without_salt(data, key)
- cipher = OpenSSL::Cipher.new("aes-128-ecb")
- cipher.encrypt
- cipher.key = key
-
- io = IO::Memory.new
- io.write(cipher.update(data))
- io.write(cipher.final)
- io.rewind
-
- return io
-end
-
-def invidious_companion_encrypt(data)
- timestamp = Time.utc.to_unix
- encrypted_data = encrypt_ecb_without_salt("#{timestamp}|#{data}", CONFIG.invidious_companion_key)
- return Base64.urlsafe_encode(encrypted_data)
-end
diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr
deleted file mode 100644
index 260d250f..00000000
--- a/src/invidious/helpers/webvtt.cr
+++ /dev/null
@@ -1,81 +0,0 @@
-# Namespace for logic relating to generating WebVTT files
-#
-# Probably not compliant to WebVTT's specs but it is enough for Invidious.
-module WebVTT
- # A WebVTT builder generates WebVTT files
- private class Builder
- # See https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#cue_payload
- private ESCAPE_SUBSTITUTIONS = {
- '&' => "&",
- '<' => "<",
- '>' => ">",
- '\u200E' => "",
- '\u200F' => "",
- '\u00A0' => " ",
- }
-
- def initialize(@io : IO)
- end
-
- # Writes an vtt cue with the specified time stamp and contents
- def cue(start_time : Time::Span, end_time : Time::Span, text : String)
- timestamp(start_time, end_time)
- @io << self.escape(text)
- @io << "\n\n"
- end
-
- private def timestamp(start_time : Time::Span, end_time : Time::Span)
- timestamp_component(start_time)
- @io << " --> "
- timestamp_component(end_time)
-
- @io << '\n'
- end
-
- private def timestamp_component(timestamp : Time::Span)
- @io << timestamp.hours.to_s.rjust(2, '0')
- @io << ':' << timestamp.minutes.to_s.rjust(2, '0')
- @io << ':' << timestamp.seconds.to_s.rjust(2, '0')
- @io << '.' << timestamp.milliseconds.to_s.rjust(3, '0')
- end
-
- private def escape(text : String) : String
- return text.gsub(ESCAPE_SUBSTITUTIONS)
- end
-
- def document(setting_fields : Hash(String, String)? = nil, &)
- @io << "WEBVTT\n"
-
- if setting_fields
- setting_fields.each do |name, value|
- @io << name << ": " << value << '\n'
- end
- end
-
- @io << '\n'
-
- yield
- end
- end
-
- # Returns the resulting `String` of writing WebVTT to the yielded `WebVTT::Builder`
- #
- # ```
- # string = WebVTT.build do |vtt|
- # vtt.cue(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1")
- # vtt.cue(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2")
- # end
- #
- # string # => "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nLine 1\n\n00:00:02.000 --> 00:00:03.000\nLine 2\n\n"
- # ```
- #
- # Accepts an optional settings fields hash to add settings attribute to the resulting vtt file.
- def self.build(setting_fields : Hash(String, String)? = nil, &)
- String.build do |str|
- builder = Builder.new(str)
- builder.document(setting_fields) do
- yield builder
- end
- end
- end
-end
diff --git a/src/invidious/http_server/utils.cr b/src/invidious/http_server/utils.cr
deleted file mode 100644
index 623a9177..00000000
--- a/src/invidious/http_server/utils.cr
+++ /dev/null
@@ -1,41 +0,0 @@
-require "uri"
-
-module Invidious::HttpServer
- module Utils
- extend self
-
- def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
- url = URI.parse(raw_url)
-
- # Add some URL parameters
- params = url.query_params
- params["host"] = url.host.not_nil! # Should never be nil, in theory
- params["region"] = region if !region.nil?
- url.query_params = params
-
- if absolute
- return "#{HOST_URL}#{url.request_target}"
- else
- return url.request_target
- end
- end
-
- def add_params_to_url(url : String | URI, params : URI::Params) : URI
- url = URI.parse(url) if url.is_a?(String)
-
- url_query = url.query || ""
-
- # Append the parameters
- url.query = String.build do |str|
- if !url_query.empty?
- str << url_query
- str << '&'
- end
-
- str << params
- end
-
- return url
- end
- end
-end
diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr
index b6b673f7..ec0cad64 100644
--- a/src/invidious/jobs.cr
+++ b/src/invidious/jobs.cr
@@ -1,39 +1,12 @@
module Invidious::Jobs
JOBS = [] of BaseJob
- # Automatically generate a structure that wraps the various
- # jobs' configs, so that the following YAML config can be used:
- #
- # jobs:
- # job_name:
- # enabled: true
- # some_property: "value"
- #
- macro finished
- struct JobsConfig
- include YAML::Serializable
-
- {% for sc in BaseJob.subclasses %}
- # Voodoo macro to transform `Some::Module::CustomJob` to `custom`
- {% class_name = sc.id.split("::").last.id.gsub(/Job$/, "").underscore %}
-
- getter {{ class_name }} = {{ sc.name }}::Config.new
- {% end %}
-
- def initialize
- end
- end
- end
-
def self.register(job : BaseJob)
JOBS << job
end
def self.start_all
JOBS.each do |job|
- # Don't run the main rountine if the job is disabled by config
- next if job.disabled?
-
spawn { job.begin }
end
end
diff --git a/src/invidious/jobs/base_job.cr b/src/invidious/jobs/base_job.cr
index f90f0bfe..47e75864 100644
--- a/src/invidious/jobs/base_job.cr
+++ b/src/invidious/jobs/base_job.cr
@@ -1,33 +1,3 @@
abstract class Invidious::Jobs::BaseJob
abstract def begin
-
- # When this base job class is inherited, make sure to define
- # a basic "Config" structure, that contains the "enable" property,
- # and to create the associated instance property.
- #
- macro inherited
- macro finished
- # This config structure can be expanded as required.
- struct Config
- include YAML::Serializable
-
- property enable = true
-
- def initialize
- end
- end
-
- property cfg = Config.new
-
- # Return true if job is enabled by config
- protected def enabled? : Bool
- return (@cfg.enable == true)
- end
-
- # Return true if job is disabled by config
- protected def disabled? : Bool
- return (@cfg.enable == false)
- end
- end
- end
end
diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr
new file mode 100644
index 00000000..71f8a938
--- /dev/null
+++ b/src/invidious/jobs/bypass_captcha_job.cr
@@ -0,0 +1,135 @@
+class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
+ def begin
+ loop do
+ begin
+ random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String})
+ if !random_video
+ random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"}
+ end
+ {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path|
+ response = YT_POOL.client &.get(path)
+ if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
+ html = XML.parse_html(response.body)
+ form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
+ site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
+ s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
+
+ inputs = {} of String => String
+ form.xpath_nodes(%(.//input[@name])).map do |node|
+ inputs[node["name"]] = node["value"]
+ end
+
+ headers = response.cookies.add_request_headers(HTTP::Headers.new)
+
+ response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask",
+ headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
+ "clientKey" => CONFIG.captcha_key,
+ "task" => {
+ "type" => "NoCaptchaTaskProxyless",
+ "websiteURL" => "https://www.youtube.com#{path}",
+ "websiteKey" => site_key,
+ "recaptchaDataSValue" => s_value,
+ },
+ }.to_json).body)
+
+ raise response["error"].as_s if response["error"]?
+ task_id = response["taskId"].as_i
+
+ loop do
+ sleep 10.seconds
+
+ response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult",
+ headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
+ "clientKey" => CONFIG.captcha_key,
+ "taskId" => task_id,
+ }.to_json).body)
+
+ if response["status"]?.try &.== "ready"
+ break
+ elsif response["errorId"]?.try &.as_i != 0
+ raise response["errorDescription"].as_s
+ end
+ end
+
+ inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
+ headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
+ response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
+
+ response.cookies
+ .select { |cookie| cookie.name != "PREF" }
+ .each { |cookie| CONFIG.cookies << cookie }
+
+ # Persist cookies between runs
+ File.write("config/config.yml", CONFIG.to_yaml)
+ elsif response.headers["Location"]?.try &.includes?("/sorry/index")
+ location = response.headers["Location"].try { |u| URI.parse(u) }
+ headers = HTTP::Headers{":authority" => location.host.not_nil!}
+ response = YT_POOL.client &.get(location.request_target, headers)
+
+ html = XML.parse_html(response.body)
+ form = html.xpath_node(%(//form[@action="index"])).not_nil!
+ site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
+ s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
+
+ inputs = {} of String => String
+ form.xpath_nodes(%(.//input[@name])).map do |node|
+ inputs[node["name"]] = node["value"]
+ end
+
+ captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url))
+ captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
+ response = JSON.parse(captcha_client.post("/createTask",
+ headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
+ "clientKey" => CONFIG.captcha_key,
+ "task" => {
+ "type" => "NoCaptchaTaskProxyless",
+ "websiteURL" => location.to_s,
+ "websiteKey" => site_key,
+ "recaptchaDataSValue" => s_value,
+ },
+ }.to_json).body)
+
+ captcha_client.close
+
+ raise response["error"].as_s if response["error"]?
+ task_id = response["taskId"].as_i
+
+ loop do
+ sleep 10.seconds
+
+ response = JSON.parse(captcha_client.post("/getTaskResult",
+ headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
+ "clientKey" => CONFIG.captcha_key,
+ "taskId" => task_id,
+ }.to_json).body)
+
+ if response["status"]?.try &.== "ready"
+ break
+ elsif response["errorId"]?.try &.as_i != 0
+ raise response["errorDescription"].as_s
+ end
+ end
+
+ inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
+ headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
+ response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
+ headers = HTTP::Headers{
+ "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
+ }
+ cookies = HTTP::Cookies.from_client_headers(headers)
+
+ cookies.each { |cookie| CONFIG.cookies << cookie }
+
+ # Persist cookies between runs
+ File.write("config/config.yml", CONFIG.to_yaml)
+ end
+ end
+ rescue ex
+ LOGGER.error("BypassCaptchaJob: #{ex.message}")
+ ensure
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+ end
+end
diff --git a/src/invidious/jobs/clear_expired_items_job.cr b/src/invidious/jobs/clear_expired_items_job.cr
deleted file mode 100644
index 17191aac..00000000
--- a/src/invidious/jobs/clear_expired_items_job.cr
+++ /dev/null
@@ -1,27 +0,0 @@
-class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob
- # Remove items (videos, nonces, etc..) whose cache is outdated every hour.
- # Removes the need for a cron job.
- def begin
- loop do
- failed = false
-
- LOGGER.info("jobs: running ClearExpiredItems job")
-
- begin
- Invidious::Database::Videos.delete_expired
- Invidious::Database::Nonces.delete_expired
- rescue DB::Error
- failed = true
- end
-
- # Retry earlier than scheduled on DB error
- if failed
- LOGGER.info("jobs: ClearExpiredItems failed. Retrying in 10 minutes.")
- sleep 10.minutes
- else
- LOGGER.info("jobs: ClearExpiredItems done.")
- sleep 1.hour
- end
- end
- end
-end
diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr
deleted file mode 100644
index cb4280b9..00000000
--- a/src/invidious/jobs/instance_refresh_job.cr
+++ /dev/null
@@ -1,97 +0,0 @@
-class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob
- # We update the internals of a constant as so it can be accessed from anywhere
- # within the codebase
- #
- # "INSTANCES" => Array(Tuple(String, String)) # region, instance
-
- INSTANCES = {"INSTANCES" => [] of Tuple(String, String)}
-
- def initialize
- end
-
- def begin
- loop do
- refresh_instances
- LOGGER.info("InstanceListRefreshJob: Done, sleeping for 30 minutes")
- sleep 30.minute
- Fiber.yield
- end
- end
-
- # Refreshes the list of instances used for redirects.
- #
- # Does the following three checks for each instance
- # - Is it a clear-net instance?
- # - Is it an instance with a good uptime?
- # - Is it an updated instance?
- private def refresh_instances
- raw_instance_list = self.fetch_instances
- filtered_instance_list = [] of Tuple(String, String)
-
- raw_instance_list.each do |instance_data|
- # TODO allow Tor hidden service instances when the current instance
- # is also a hidden service. Same for i2p and any other non-clearnet instances.
- begin
- domain = instance_data[0]
- info = instance_data[1]
- stats = info["stats"]
-
- next unless info["type"] == "https"
- next if bad_uptime?(info["monitor"])
- next if outdated?(stats["software"]["version"])
-
- filtered_instance_list << {info["region"].as_s, domain.as_s}
- rescue ex
- if domain
- LOGGER.info("InstanceListRefreshJob: failed to parse information from '#{domain}' because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ")
- else
- LOGGER.info("InstanceListRefreshJob: failed to parse information from an instance because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ")
- end
- end
- end
-
- if !filtered_instance_list.empty?
- INSTANCES["INSTANCES"] = filtered_instance_list
- end
- end
-
- # Fetches information regarding instances from api.invidious.io or an otherwise configured URL
- private def fetch_instances : Array(JSON::Any)
- begin
- # We directly call the stdlib HTTP::Client here as it allows us to negate the effects
- # of the force_resolve config option. This is needed as api.invidious.io does not support ipv6
- # and as such the following request raises if we were to use force_resolve with the ipv6 value.
- instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io"))
-
- # Timeouts
- instance_api_client.connect_timeout = 10.seconds
- instance_api_client.dns_timeout = 10.seconds
-
- raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a
- instance_api_client.close
- rescue ex : Socket::ConnectError | IO::TimeoutError | JSON::ParseException
- raw_instance_list = [] of JSON::Any
- end
-
- return raw_instance_list
- end
-
- # Checks if the given target instance is outdated
- private def outdated?(target_instance_version) : Bool
- remote_commit_date = target_instance_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/)
- return false if !remote_commit_date
-
- remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC)
- local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC)
-
- return (remote_commit_date - local_commit_date).abs.days > 30
- end
-
- # Checks if the uptime of the target instance is greater than 90% over a 30 day period
- private def bad_uptime?(target_instance_health_monitor) : Bool
- return true if !target_instance_health_monitor["down"].as_bool == false
- return true if target_instance_health_monitor["uptime"].as_f < 90
-
- return false
- end
-end
diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr
index 968ee47f..2f525e08 100644
--- a/src/invidious/jobs/notification_job.cr
+++ b/src/invidious/jobs/notification_job.cr
@@ -1,103 +1,15 @@
-struct VideoNotification
- getter video_id : String
- getter channel_id : String
- getter published : Time
-
- def_hash @channel_id, @video_id
-
- def ==(other)
- video_id == other.video_id
- end
-
- def self.from_video(video : ChannelVideo) : self
- VideoNotification.new(video.id, video.ucid, video.published)
- end
-
- def initialize(@video_id, @channel_id, @published)
- end
-
- def clone : VideoNotification
- VideoNotification.new(video_id.clone, channel_id.clone, published.clone)
- end
-end
-
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
- private getter notification_channel : ::Channel(VideoNotification)
- private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
+ private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
private getter pg_url : URI
- def initialize(@notification_channel, @connection_channel, @pg_url)
+ def initialize(@connection_channel, @pg_url)
end
def begin
- connections = [] of ::Channel(PQ::Notification)
+ connections = [] of Channel(PQ::Notification)
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
- # hash of channels to their videos (id+published) that need notifying
- to_notify = Hash(String, Set(VideoNotification)).new(
- ->(hash : Hash(String, Set(VideoNotification)), key : String) {
- hash[key] = Set(VideoNotification).new
- }
- )
- notify_mutex = Mutex.new
-
- # fiber to locally cache all incoming notifications (from pubsub webhooks and refresh channels job)
- spawn do
- begin
- loop do
- notification = notification_channel.receive
- notify_mutex.synchronize do
- to_notify[notification.channel_id] << notification
- end
- end
- end
- end
- # fiber to regularly persist all cached notifications
- spawn do
- loop do
- begin
- LOGGER.debug("NotificationJob: waking up")
- cloned = {} of String => Set(VideoNotification)
- notify_mutex.synchronize do
- cloned = to_notify.clone
- to_notify.clear
- end
-
- cloned.each do |channel_id, notifications|
- if notifications.empty?
- next
- end
-
- LOGGER.info("NotificationJob: updating channel #{channel_id} with #{notifications.size} notifications")
- if CONFIG.enable_user_notifications
- video_ids = notifications.map(&.video_id)
- Invidious::Database::Users.add_multiple_notifications(channel_id, video_ids)
- PG_DB.using_connection do |conn|
- notifications.each do |n|
- # Deliver notifications to `/api/v1/auth/notifications`
- payload = {
- "topic" => n.channel_id,
- "videoId" => n.video_id,
- "published" => n.published.to_unix,
- }.to_json
- conn.exec("NOTIFY notifications, E'#{payload}'")
- end
- end
- else
- Invidious::Database::Users.feed_needs_update(channel_id)
- end
- end
-
- LOGGER.trace("NotificationJob: Done, sleeping")
- rescue ex
- LOGGER.error("NotificationJob: #{ex.message}")
- end
- sleep 1.minute
- Fiber.yield
- end
- end
-
loop do
action, connection = connection_channel.receive
diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr
index 80812a63..92681408 100644
--- a/src/invidious/jobs/refresh_channels_job.cr
+++ b/src/invidious/jobs/refresh_channels_job.cr
@@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
max_fibers = CONFIG.channel_threads
lim_fibers = max_fibers
active_fibers = 0
- active_channel = ::Channel(Bool).new
+ active_channel = Channel(Bool).new
backoff = 2.minutes
loop do
diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr
index 4f8130df..4b52c959 100644
--- a/src/invidious/jobs/refresh_feeds_job.cr
+++ b/src/invidious/jobs/refresh_feeds_job.cr
@@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
- active_channel = ::Channel(Bool).new
+ active_channel = Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr
index 66c91ad5..a113bd77 100644
--- a/src/invidious/jobs/statistics_refresh_job.cr
+++ b/src/invidious/jobs/statistics_refresh_job.cr
@@ -18,13 +18,6 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => 0_i64,
},
-
- #
- # "totalRequests" => 0_i64,
- # "successfulRequests" => 0_i64
- # "ratio" => 0_i64
- #
- "playback" => {} of String => Int64 | Float64,
}
private getter db : DB::Database
@@ -37,7 +30,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
loop do
refresh_stats
- sleep 10.minute
+ sleep 1.minute
Fiber.yield
end
end
@@ -56,15 +49,12 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
users["total"] = Invidious::Database::Statistics.count_users_total
- users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m
- users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m
+ users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m
+ users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m
STATISTICS["metadata"] = {
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,
}
-
- # Reset playback requests tracker
- STATISTICS["playback"] = {} of String => Int64 | Float64
end
end
diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr
index 8584fb9c..a431a48a 100644
--- a/src/invidious/jobs/subscribe_to_feeds_job.cr
+++ b/src/invidious/jobs/subscribe_to_feeds_job.cr
@@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
end
active_fibers = 0
- active_channel = ::Channel(Bool).new
+ active_channel = Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
diff --git a/src/invidious/jobs/update_decrypt_function_job.cr b/src/invidious/jobs/update_decrypt_function_job.cr
new file mode 100644
index 00000000..6fa0ae1b
--- /dev/null
+++ b/src/invidious/jobs/update_decrypt_function_job.cr
@@ -0,0 +1,14 @@
+class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
+ def begin
+ loop do
+ begin
+ DECRYPT_FUNCTION.update_decrypt_function
+ rescue ex
+ LOGGER.error("UpdateDecryptFunctionJob : #{ex.message}")
+ ensure
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+ end
+end
diff --git a/src/invidious/jsonify/api_v1/common.cr b/src/invidious/jsonify/api_v1/common.cr
deleted file mode 100644
index 64b06465..00000000
--- a/src/invidious/jsonify/api_v1/common.cr
+++ /dev/null
@@ -1,18 +0,0 @@
-require "json"
-
-module Invidious::JSONify::APIv1
- extend self
-
- def thumbnails(json : JSON::Builder, id : String)
- json.array do
- build_thumbnails(id).each do |thumbnail|
- json.object do
- json.field "quality", thumbnail[:name]
- json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
- json.field "width", thumbnail[:width]
- json.field "height", thumbnail[:height]
- end
- end
- end
- end
-end
diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr
deleted file mode 100644
index 58805af2..00000000
--- a/src/invidious/jsonify/api_v1/video_json.cr
+++ /dev/null
@@ -1,301 +0,0 @@
-require "json"
-
-module Invidious::JSONify::APIv1
- extend self
-
- def video(video : Video, json : JSON::Builder, *, locale : String?, proxy : Bool = false)
- json.object do
- json.field "type", video.video_type
-
- json.field "title", video.title
- json.field "videoId", video.id
-
- json.field "error", video.info["reason"] if video.info["reason"]?
-
- json.field "videoThumbnails" do
- self.thumbnails(json, video.id)
- end
- json.field "storyboards" do
- self.storyboards(json, video.id, video.storyboards)
- end
-
- json.field "description", video.description
- json.field "descriptionHtml", video.description_html
- json.field "published", video.published.to_unix
- json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale))
- json.field "keywords", video.keywords
-
- json.field "viewCount", video.views
- json.field "likeCount", video.likes
- json.field "dislikeCount", 0_i64
-
- json.field "paid", video.paid
- json.field "premium", video.premium
- json.field "isFamilyFriendly", video.is_family_friendly
- json.field "allowedRegions", video.allowed_regions
- json.field "genre", video.genre
- json.field "genreUrl", video.genre_url
-
- json.field "author", video.author
- json.field "authorId", video.ucid
- json.field "authorUrl", "/channel/#{video.ucid}"
- json.field "authorVerified", video.author_verified
-
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", video.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
-
- json.field "subCountText", video.sub_count_text
-
- json.field "lengthSeconds", video.length_seconds
- json.field "allowRatings", video.allow_ratings
- json.field "rating", 0_i64
- json.field "isListed", video.is_listed
- json.field "liveNow", video.live_now
- json.field "isPostLiveDvr", video.post_live_dvr
- json.field "isUpcoming", video.upcoming?
-
- if video.premiere_timestamp
- json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix
- end
-
- if hlsvp = video.hls_manifest_url
- hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
- json.field "hlsUrl", hlsvp
- end
-
- json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{video.id}"
-
- json.field "adaptiveFormats" do
- json.array do
- video.adaptive_fmts.each do |fmt|
- json.object do
- # Only available on regular videos, not livestreams/OTF streams
- if init_range = fmt["initRange"]?
- json.field "init", "#{init_range["start"]}-#{init_range["end"]}"
- end
- if index_range = fmt["indexRange"]?
- json.field "index", "#{index_range["start"]}-#{index_range["end"]}"
- end
-
- # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only)
- json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
-
- if proxy
- json.field "url", Invidious::HttpServer::Utils.proxy_video_url(
- fmt["url"].to_s, absolute: true
- )
- else
- json.field "url", fmt["url"]
- end
-
- json.field "itag", fmt["itag"].as_i.to_s
- json.field "type", fmt["mimeType"]
- json.field "clen", fmt["contentLength"]? || "-1"
-
- # Last modified is a unix timestamp with µS, with the dot omitted.
- # E.g: 1638056732(.)141582
- #
- # On livestreams, it's not present, so always fall back to the
- # current unix timestamp (up to mS precision) for compatibility.
- last_modified = fmt["lastModified"]?
- last_modified ||= "#{Time.utc.to_unix_ms}000"
- json.field "lmt", last_modified
-
- json.field "projectionType", fmt["projectionType"]
-
- height = fmt["height"]?.try &.as_i
- width = fmt["width"]?.try &.as_i
-
- fps = fmt["fps"]?.try &.as_i
-
- if fps
- json.field "fps", fps
- end
-
- if height && width
- json.field "size", "#{width}x#{height}"
- json.field "resolution", "#{height}p"
-
- quality_label = "#{width > height ? height : width}p"
-
- if fps && fps > 30
- quality_label += fps.to_s
- end
-
- json.field "qualityLabel", quality_label
- end
-
- if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
- json.field "container", fmt_info["ext"]
- json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
- end
-
- # Livestream chunk infos
- json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec")
- json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec")
-
- # Audio-related data
- json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality")
- json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate")
- json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels")
-
- # Extra misc stuff
- json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo")
- json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack")
- end
- end
- end
- end
-
- json.field "formatStreams" do
- json.array do
- video.fmt_stream.each do |fmt|
- json.object do
- if proxy
- json.field "url", Invidious::HttpServer::Utils.proxy_video_url(
- fmt["url"].to_s, absolute: true
- )
- else
- json.field "url", fmt["url"]
- end
- json.field "itag", fmt["itag"].as_i.to_s
- json.field "type", fmt["mimeType"]
- json.field "quality", fmt["quality"]
-
- json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
-
- height = fmt["height"]?.try &.as_i
- width = fmt["width"]?.try &.as_i
-
- fps = fmt["fps"]?.try &.as_i
-
- if fps
- json.field "fps", fps
- end
-
- if height && width
- json.field "size", "#{width}x#{height}"
- json.field "resolution", "#{height}p"
-
- quality_label = "#{width > height ? height : width}p"
-
- if fps && fps > 30
- quality_label += fps.to_s
- end
-
- json.field "qualityLabel", quality_label
- end
-
- if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
- json.field "container", fmt_info["ext"]
- json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
- end
- end
- end
- end
- end
-
- json.field "captions" do
- json.array do
- video.captions.each do |caption|
- json.object do
- json.field "label", caption.name
- json.field "language_code", caption.language_code
- json.field "url", "/api/v1/captions/#{video.id}?label=#{URI.encode_www_form(caption.name)}"
- end
- end
- end
- end
-
- if !video.music.empty?
- json.field "musicTracks" do
- json.array do
- video.music.each do |music|
- json.object do
- json.field "song", music.song
- json.field "artist", music.artist
- json.field "album", music.album
- json.field "license", music.license
- end
- end
- end
- end
- end
-
- json.field "recommendedVideos" do
- json.array do
- video.related_videos.each do |rv|
- if rv["id"]?
- json.object do
- json.field "videoId", rv["id"]
- json.field "title", rv["title"]
- json.field "videoThumbnails" do
- self.thumbnails(json, rv["id"])
- end
-
- json.field "author", rv["author"]
- json.field "authorUrl", "/channel/#{rv["ucid"]?}"
- json.field "authorId", rv["ucid"]?
- json.field "authorVerified", rv["author_verified"] == "true"
- if rv["author_thumbnail"]?
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
- end
-
- json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
- json.field "viewCountText", rv["short_view_count"]?
- json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
- json.field "published", rv["published"]?
- if rv["published"]?.try &.presence
- json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale))
- else
- json.field "publishedText", ""
- end
- end
- end
- end
- end
- end
- end
- end
-
- def storyboards(json, id, storyboards)
- json.array do
- storyboards.each do |sb|
- json.object do
- json.field "url", "/api/v1/storyboards/#{id}?width=#{sb.width}&height=#{sb.height}"
- json.field "templateUrl", sb.url.to_s
- json.field "width", sb.width
- json.field "height", sb.height
- json.field "count", sb.count
- json.field "interval", sb.interval
- json.field "storyboardWidth", sb.columns
- json.field "storyboardHeight", sb.rows
- json.field "storyboardCount", sb.images_count
- end
- end
- end
- end
-end
diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr
index 28ff0ff6..3f342b92 100644
--- a/src/invidious/mixes.cr
+++ b/src/invidious/mixes.cr
@@ -81,7 +81,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
})
end
-def template_mix(mix, listen)
+def template_mix(mix)
html = <<-END_HTML
@@ -95,9 +95,9 @@ def template_mix(mix, listen)
mix["videos"].as_a.each do |video|
html += <<-END_HTML
-
-
- END_HTML
- else
- html << <<-END_HTML
- ![]()
)
- end
- end
- html << <<-END_HTML
- "
@@ -48,14 +42,15 @@ module Invidious::Frontend::WatchPage
str << translate(locale, "Download as: ")
str << "\n"
- str << "\t\t
END_HTML
@@ -106,7 +95,7 @@ end
# -------------------
macro error_atom(*args)
- error_atom_helper(env, {{args.splat}})
+ error_atom_helper(env, {{*args}})
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
@@ -132,14 +121,14 @@ end
# -------------------
macro error_json(*args)
- error_json_helper(env, {{args.splat}})
+ error_json_helper(env, {{*args}})
end
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
exception : Exception,
- additional_fields : Hash(String, Object) | Nil = nil,
+ additional_fields : Hash(String, Object) | Nil = nil
)
if exception.is_a?(InfoException)
return error_json_helper(env, status_code, exception.message || "", additional_fields)
@@ -161,7 +150,7 @@ def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
message : String,
- additional_fields : Hash(String, Object) | Nil = nil,
+ additional_fields : Hash(String, Object) | Nil = nil
)
env.response.content_type = "application/json"
env.response.status_code = status_code
@@ -201,7 +190,7 @@ def error_redirect_helper(env : HTTP::Server::Context)
#{switch_instance}
-
- END_HTML
- end
- end
- end
- end
-end
diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr
deleted file mode 100644
index a0e1d783..00000000
--- a/src/invidious/frontend/comments_youtube.cr
+++ /dev/null
@@ -1,208 +0,0 @@
-module Invidious::Frontend::Comments
- extend self
-
- def template_youtube(comments, locale, thin_mode, is_replies = false)
- String.build do |html|
- root = comments["comments"].as_a
- root.each do |child|
- if child["replies"]?
- replies_count_text = translate_count(locale,
- "comments_view_x_replies",
- child["replies"]["replyCount"].as_i64 || 0,
- NumberFormatting::Separator
- )
-
- replies_html = <<-END_HTML
-
- END_HTML
- end
-
- html << <<-END_HTML
-
- - [ − ] - #{child.author} - #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} - #{translate(locale, "`x` ago", recode_date(child.created_utc, locale))} - #{translate(locale, "permalink")} -
-
- #{body_html}
- #{replies_html}
-
-
-
-
- END_HTML
- elsif comments["authorId"]? && !comments["singlePost"]?
- # for posts we should display a link to the post
- replies_count_text = translate_count(locale,
- "comments_view_x_replies",
- child["replyCount"].as_i64 || 0,
- NumberFormatting::Separator
- )
-
- replies_html = <<-END_HTML
-
-
-
-
-
-
- END_HTML
- end
-
- if !thin_mode
- author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).request_target}"
- else
- author_thumbnail = ""
- end
-
- author_name = HTML.escape(child["author"].as_s)
- sponsor_icon = ""
- if child["verified"]?.try &.as_bool && child["authorIsChannelOwner"]?.try &.as_bool
- author_name += " "
- elsif child["verified"]?.try &.as_bool
- author_name += " "
- end
-
- if child["isSponsor"]?.try &.as_bool
- sponsor_icon = String.build do |str|
- str << %(
-
-
-
-
- END_HTML
- end
-
- if comments["continuation"]?
- html << <<-END_HTML
-
-
-
-
-
- #{translate(locale, "carousel_skip")}
-
- END_HTML
- else nil # Ignore
- end
- end
-
- html << <<-END_HTML
-
- - - #{author_name} - - #{sponsor_icon} -
#{child["contentHtml"]}
- END_HTML - - if child["attachment"]? - attachment = child["attachment"] - - case attachment["type"] - when "image" - attachment = attachment["imageThumbnails"][1] - - html << <<-END_HTML -
-
- END_HTML
- when "video"
- if attachment["error"]?
- html << <<-END_HTML
-
-
-
-
-
-
- END_HTML
- else
- html << <<-END_HTML
- #{attachment["error"]}
-
-
-
- END_HTML
- end
- when "multiImage"
- html << <<-END_HTML
-
- END_HTML
- image_array = attachment["images"].as_a
-
- image_array.each_index do |i|
- html << <<-END_HTML
-
- (i + 1).to_s, "total" => image_array.size.to_s})}" tabindex="0">
- 
-
- END_HTML
- end
-
- html << <<-END_HTML
-
-
- END_HTML
- attachment["images"].as_a.each_index do |i|
- html << <<-END_HTML
-
- END_HTML
- end
- html << <<-END_HTML
-
-
-
- #{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}
- |
- END_HTML
-
- if comments["videoId"]?
- html << <<-END_HTML
- [YT]
- |
- END_HTML
- elsif comments["authorId"]?
- html << <<-END_HTML
- [YT]
- |
- END_HTML
- end
-
- html << <<-END_HTML
- #{number_with_separator(child["likeCount"])}
- END_HTML
-
- if child["creatorHeart"]?
- if !thin_mode
- creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).request_target}"
- else
- creator_thumbnail = ""
- end
-
- html << <<-END_HTML
-
-
-
-
-
-
-
-
-
- END_HTML
- end
-
- html << <<-END_HTML
-
-
-
- END_HTML
- end
- end
- end
-end
diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr
index 7a6cf79d..43ba9f5c 100644
--- a/src/invidious/frontend/misc.cr
+++ b/src/invidious/frontend/misc.cr
@@ -6,9 +6,9 @@ module Invidious::Frontend::Misc
if prefs.automatic_instance_redirect
current_page = env.get?("current_page").as(String)
- return "/redirect?referer=#{current_page}"
+ redirect_url = "/redirect?referer=#{current_page}"
else
- return "https://redirect.invidious.io#{env.request.resource}"
+ redirect_url = "https://redirect.invidious.io#{env.request.resource}"
end
end
end
diff --git a/src/invidious/frontend/pagination.cr b/src/invidious/frontend/pagination.cr
deleted file mode 100644
index a29f5936..00000000
--- a/src/invidious/frontend/pagination.cr
+++ /dev/null
@@ -1,121 +0,0 @@
-require "uri"
-
-module Invidious::Frontend::Pagination
- extend self
-
- private def first_page(str : String::Builder, locale : String?, url : String)
- str << %()
-
- if locale_is_rtl?(locale)
- # Inverted arrow ("first" points to the right)
- str << translate(locale, "First page")
- str << " "
- str << %()
- else
- # Regular arrow ("first" points to the left)
- str << %()
- str << " "
- str << translate(locale, "First page")
- end
-
- str << ""
- end
-
- private def previous_page(str : String::Builder, locale : String?, url : String)
- # Link
- str << %()
-
- if locale_is_rtl?(locale)
- # Inverted arrow ("previous" points to the right)
- str << translate(locale, "Previous page")
- str << " "
- str << %()
- else
- # Regular arrow ("previous" points to the left)
- str << %()
- str << " "
- str << translate(locale, "Previous page")
- end
-
- str << ""
- end
-
- private def next_page(str : String::Builder, locale : String?, url : String)
- # Link
- str << %()
-
- if locale_is_rtl?(locale)
- # Inverted arrow ("next" points to the left)
- str << %()
- str << " "
- str << translate(locale, "Next page")
- else
- # Regular arrow ("next" points to the right)
- str << translate(locale, "Next page")
- str << " "
- str << %()
- end
-
- str << ""
- end
-
- def nav_numeric(locale : String?, *, base_url : String | URI, current_page : Int, show_next : Bool = true)
- return String.build do |str|
- str << %(\n)
- str << %(
\n\n)
- end
- end
-
- def nav_ctoken(locale : String?, *, base_url : String | URI, ctoken : String?, first_page : Bool, params : URI::Params)
- return String.build do |str|
- str << %(\n)
-
- str << %(
\n)
- str << %()
-
- if current_page > 1
- params_prev = URI::Params{"page" => (current_page - 1).to_s}
- url_prev = HttpServer::Utils.add_params_to_url(base_url, params_prev)
-
- self.previous_page(str, locale, url_prev.to_s)
- end
-
- str << %(
\n)
- str << %(\n)
-
- str << %(\n)
- str << %(
\n\n)
- end
- end
-end
diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr
index 15d925e3..80b67641 100644
--- a/src/invidious/frontend/watch_page.cr
+++ b/src/invidious/frontend/watch_page.cr
@@ -7,32 +7,26 @@ module Invidious::Frontend::WatchPage
getter full_videos : Array(Hash(String, JSON::Any))
getter video_streams : Array(Hash(String, JSON::Any))
getter audio_streams : Array(Hash(String, JSON::Any))
- getter captions : Array(Invidious::Videos::Captions::Metadata)
+ getter captions : Array(Caption)
def initialize(
@full_videos,
@video_streams,
@audio_streams,
- @captions,
+ @captions
)
end
end
def download_widget(locale : String, video : Video, video_assets : VideoAssets) : String
if CONFIG.disabled?("downloads")
- return "\n)
-
- str << %(
\n)
- str << %()
-
- if !first_page
- self.first_page(str, locale, base_url.to_s)
- end
-
- str << %(
\n)
-
- str << %(\n)
-
- str << %(#{translate(locale, "Download is disabled")}
" - end - - url = "/download" - if (CONFIG.invidious_companion.present?) - invidious_companion = CONFIG.invidious_companion.sample - url = "#{invidious_companion.public_url}/download?check=#{invidious_companion_encrypt(video.id)}" + return "#{translate(locale, "Download is disabled.")}
" end return String.build(4000) do |str| str << ")|(
": "\n", @@ -78,6 +103,15 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -104,6 +138,15 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -137,6 +180,15 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) + if fields_text = env.params.query["fields"]? + begin + JSONFilter.filter(response, fields_text) + rescue ex + env.response.status_code = 400 + response = {"error" => ex.message} + end + end + env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -181,20 +233,3 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end - -# Fetch the playback requests tracker from the statistics endpoint. -# -# Creates a new tracker when unavailable. -def get_playback_statistic - if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? - tracker = { - "totalRequests" => 0_i64, - "successfulRequests" => 0_i64, - "ratio" => 0_f64, - } - - Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker - end - - return tracker.as(Hash(String, Int64 | Float64)) -end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index bca2edda..fd86594c 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,22 +1,9 @@ -# Languages requiring a better level of translation (at least 20%) -# to be added to the list below: -# -# "af" => "", # Afrikaans -# "az" => "", # Azerbaijani -# "be" => "", # Belarusian -# "bn_BD" => "", # Bengali (Bangladesh) -# "ia" => "", # Interlingua -# "or" => "", # Odia -# "tk" => "", # Turkmen -# "tok => "", # Toki Pona -# +# "bn_BD" => load_locale("bn_BD"), # Bengali (Bangladesh) [Incomplete] +# "eu" => load_locale("eu"), # Basque [Incomplete] +# "sk" => load_locale("sk"), # Slovak [Incomplete] LOCALES_LIST = { "ar" => "العربية", # Arabic - "bg" => "български", # Bulgarian - "bn" => "বাংলা", # Bengali - "ca" => "Català", # Catalan "cs" => "Čeština", # Czech - "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek @@ -24,7 +11,6 @@ LOCALES_LIST = { "eo" => "Esperanto", # Esperanto "es" => "Español", # Spanish "et" => "Eesti keel", # Estonian - "eu" => "Euskara", # Basque "fa" => "فارسی", # Persian "fi" => "Suomi", # Finnish "fr" => "Français", # French @@ -37,7 +23,6 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean - "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch @@ -47,14 +32,11 @@ LOCALES_LIST = { "pt-PT" => "Português de Portugal", # Portuguese (Portugal) "ro" => "Română", # Romanian "ru" => "Русский", # Russian - "si" => "සිංහල", # Sinhala - "sk" => "Slovenčina", # Slovak "sl" => "Slovenščina", # Slovenian "sq" => "Shqip", # Albanian "sr" => "Srpski (latinica)", # Serbian (Latin) "sr_Cyrl" => "Српски (ћирилица)", # Serbian (Cyrillic) "sv-SE" => "Svenska", # Swedish - "ta" => "தமிழ்", # Tamil "tr" => "Türkçe", # Turkish "uk" => "Українська", # Ukrainian "vi" => "Tiếng Việt", # Vietnamese @@ -94,7 +76,7 @@ def load_all_locales return locales end -def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String +def translate(locale : String?, key : String, text : String | Nil = nil) : String # Log a warning if "key" doesn't exist in en-US locale and return # that key as the text, so this is more or less transparent to the user. if !LOCALES["en-US"].has_key?(key) @@ -117,12 +99,10 @@ def translate(locale : String?, key : String, text : String | Hash(String, Strin match_length = 0 raw_data.as_h.each do |hash_key, value| - if text.is_a?(String) - if md = text.try &.match(/#{hash_key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size - end + if md = text.try &.match(/#{hash_key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size end end end @@ -132,13 +112,8 @@ def translate(locale : String?, key : String, text : String | Hash(String, Strin raise "Invalid translation \"#{raw_data}\"" end - if text.is_a?(String) + if text translation = translation.gsub("`x`", text) - elsif text.is_a?(Hash(String, String)) - # adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic - text.each_key do |hash_key| - translation = translation.gsub("{{#{hash_key}}}", text[hash_key]) - end end return translation @@ -188,12 +163,3 @@ def translate_bool(locale : String?, translation : Bool) return translate(locale, "No") end end - -def locale_is_rtl?(locale : String?) - # Fallback to en-US - return false if locale.nil? - - # Arabic, Persian, Hebrew - # See https://en.wikipedia.org/wiki/Right-to-left_script#List_of_RTL_scripts - return {"ar", "fa", "he"}.includes? locale -end diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index 684e6d14..e84f88fb 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -35,35 +35,27 @@ module I18next::Plurals Special_Slovenian = 21 Special_Hebrew = 22 Special_Odia = 23 - - # Mixed v3/v4 rules in Weblate - # `es`, `pt` and `pt-PT` doesn't seem to have been refreshed - # by weblate yet, but I suspect it will happen one day. - # See: https://github.com/translate/translate/issues/4873 - Special_French_Portuguese - Special_Hungarian_Serbian - Special_Spanish_Italian end private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fil", "fr", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt", "pt-BR", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", - "eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", - "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", + "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "hu", "hy", "ia", "it", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", - "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", + "ps", "pt-PT", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ - "ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ - "be", "bs", "cnr", "dz", "ru", "uk", + "be", "bs", "cnr", "dz", "hr", "ru", "sr", "uk", ], } @@ -89,13 +81,6 @@ module I18next::Plurals "ro" => PluralForms::Special_Romanian, "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, - # Mixed v3/v4 rules - "es" => PluralForms::Special_Spanish_Italian, - "fr" => PluralForms::Special_French_Portuguese, - "hr" => PluralForms::Special_Hungarian_Serbian, - "it" => PluralForms::Special_Spanish_Italian, - "pt" => PluralForms::Special_French_Portuguese, - "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -165,8 +150,9 @@ module I18next::Plurals end def get_plural_form(locale : String) : PluralForms - # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code - if !locale.matches?(/^pt-PT$/) + # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code, + # except for pt-BR and pt-PT which needs to be kept as-is. + if !locale.matches?(/^pt-(BR|PT)$/) locale = locale.split('-')[0] end @@ -188,7 +174,7 @@ module I18next::Plurals # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check # from original i18next code - private def simple_plural?(form : PluralForms) : Bool + private def is_simple_plural(form : PluralForms) : Bool case form when .single_gt_one? then return true when .single_not_one? then return true @@ -210,7 +196,7 @@ module I18next::Plurals idx = SuffixIndex.get_index(plural_form, count) # Simple plurals are handled differently in all versions (but v4) - if @simplify_plural_suffix && simple_plural?(plural_form) + if @simplify_plural_suffix && is_simple_plural(plural_form) return (idx == 1) ? "_plural" : "" end @@ -260,10 +246,6 @@ module I18next::Plurals when .special_slovenian? then return special_slovenian(count) when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) - # Mixed v3/v4 forms - when .special_spanish_italian? then return special_cldr_spanish_italian(count) - when .special_french_portuguese? then return special_cldr_french_portuguese(count) - when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count) else # default, if nothing matched above return 0_u8 @@ -525,42 +507,5 @@ module I18next::Plurals def self.special_odia(count : Int) : UInt8 return (count == 1) ? 0_u8 : 1_u8 end - - # ------------------- - # "v3.5" rules - # ------------------- - - # Plural form for Spanish & Italian languages - # - # This rule is mostly compliant to CLDR v42 - # - def self.special_cldr_spanish_italian(count : Int) : UInt8 - return 0_u8 if (count == 1) # one - return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many - return 2_u8 # other - end - - # Plural form for French and Portuguese - # - # This rule is mostly compliant to CLDR v42 - # - def self.special_cldr_french_portuguese(count : Int) : UInt8 - return 0_u8 if (count == 0 || count == 1) # one - return 1_u8 if (count % 1_000_000 == 0) # many - return 2_u8 # other - end - - # Plural form for Hungarian and Serbian - # - # This rule is mostly compliant to CLDR v42 - # - def self.special_cldr_hungarian_serbian(count : Int) : UInt8 - n_mod_10 = count % 10 - n_mod_100 = count % 100 - - return 0_u8 if (n_mod_10 == 1 && n_mod_100 != 11) # one - return 1_u8 if (2 <= n_mod_10 <= 4 && (n_mod_100 < 12 || 14 < n_mod_100)) # few - return 2_u8 # other - end end end diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr new file mode 100644 index 00000000..b8e8f96d --- /dev/null +++ b/src/invidious/helpers/json_filter.cr @@ -0,0 +1,248 @@ +module JSONFilter + alias BracketIndex = Hash(Int64, Int64) + + alias GroupedFieldsValue = String | Array(GroupedFieldsValue) + alias GroupedFieldsList = Array(GroupedFieldsValue) + + class FieldsParser + class ParseError < Exception + end + + # Returns the `Regex` pattern used to match nest groups + def self.nest_group_pattern : Regex + # uses a '.' character to match json keys as they are allowed + # to contain any unicode codepoint + /(?:|,)(?
@@ -95,9 +95,9 @@ def template_mix(mix, listen)
mix["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
- 
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr
index 7c584d15..c4eb7507 100644
--- a/src/invidious/playlists.cr
+++ b/src/invidious/playlists.cr
@@ -46,14 +46,8 @@ struct PlaylistVideo
XML.build { |xml| to_xml(xml) }
end
- def to_json(locale : String?, json : JSON::Builder)
- to_json(json)
- end
-
def to_json(json : JSON::Builder, index : Int32? = nil)
json.object do
- json.field "type", "video"
-
json.field "title", self.title
json.field "videoId", self.id
@@ -62,7 +56,7 @@ struct PlaylistVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
- Invidious::JSONify::APIv1.thumbnails(json, self.id)
+ generate_thumbnails(json, self.id)
end
if index
@@ -73,7 +67,6 @@ struct PlaylistVideo
end
json.field "lengthSeconds", self.length_seconds
- json.field "liveNow", self.live_now
end
end
@@ -96,7 +89,6 @@ struct Playlist
property views : Int64
property updated : Time
property thumbnail : String?
- property subtitle : String?
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
@@ -108,7 +100,6 @@ struct Playlist
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
- json.field "subtitle", self.subtitle
json.field "authorThumbnails" do
json.array do
@@ -270,7 +261,7 @@ end
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
- title: playlist.title[..150],
+ title: playlist.title.byte_slice(0, 150),
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
@@ -365,16 +356,12 @@ def fetch_playlist(plid : String)
updated = Time.utc
video_count = 0
- subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle"))
-
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
- elsif text.includes? "episode"
- video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
else
@@ -410,7 +397,6 @@ def fetch_playlist(plid : String)
views: views,
updated: updated,
thumbnail: thumbnail,
- subtitle: subtitle,
})
end
@@ -432,7 +418,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32,
offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
end
- videos = [] of PlaylistVideo | ProblematicTimelineItem
+ videos = [] of PlaylistVideo
until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
# 100 videos per request
@@ -448,7 +434,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32,
end
def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
- videos = [] of PlaylistVideo | ProblematicTimelineItem
+ videos = [] of PlaylistVideo
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
@@ -500,14 +486,12 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
index: index,
})
end
- rescue ex
- videos << ProblematicTimelineItem.new(parse_exception: ex)
end
return videos
end
-def template_playlist(playlist, listen)
+def template_playlist(playlist)
html = <<-END_HTML
@@ -521,9 +505,9 @@ def template_playlist(playlist, listen)
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr
index c8db207c..9bb73136 100644
--- a/src/invidious/routes/account.cr
+++ b/src/invidious/routes/account.cr
@@ -42,6 +42,11 @@ module Invidious::Routes::Account
sid = sid.as(String)
token = env.params.body["csrf_token"]?
+ # We don't store passwords for Google accounts
+ if !user.password
+ return error_template(400, "Cannot change password for Google accounts")
+ end
+
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
@@ -49,11 +54,11 @@ module Invidious::Routes::Account
end
password = env.params.body["password"]?
- if password.nil? || password.empty?
+ if !password
return error_template(401, "Password is a required field")
end
- new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
+ new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match")
@@ -198,7 +203,7 @@ module Invidious::Routes::Account
referer = get_referer(env)
if !user
- return env.redirect "/login?referer=#{URI.encode_path_segment(env.request.resource)}"
+ return env.redirect referer
end
user = user.as(User)
@@ -240,7 +245,7 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
+ scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
@@ -257,7 +262,6 @@ module Invidious::Routes::Account
end
query["token"] = access_token
- query["username"] = URI.encode_path_segment(user.email)
url.query = query.to_s
env.redirect url.to_s
@@ -328,9 +332,17 @@ module Invidious::Routes::Account
end
end
- case action = env.params.query["action"]?
- when "revoke_token"
- session = env.params.query["session"]
+ if env.params.query["action_revoke_token"]?
+ action = "action_revoke_token"
+ else
+ return env.redirect referer
+ end
+
+ session = env.params.query["session"]?
+ session ||= ""
+
+ case action
+ when .starts_with? "action_revoke_token"
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
return error_json(400, "Unsupported action #{action}")
diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr
index c27caad7..bfb8a377 100644
--- a/src/invidious/routes/api/manifest.cr
+++ b/src/invidious/routes/api/manifest.cr
@@ -8,17 +8,14 @@ module Invidious::Routes::API::Manifest
id = env.params.url["id"]
region = env.params.query["region"]?
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
- end
-
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin
video = get_video(id, region: region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
haltf env, status_code: 404
rescue ex
@@ -26,27 +23,28 @@ module Invidious::Routes::API::Manifest
end
if dashmpd = video.dash_manifest_url
- response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
+ manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body
- if response.status_code != 200
- haltf env, status_code: response.status_code
- end
+ manifest = manifest.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
+ url = baseurl.lchop("")
+ url = url.rchop(" ")
+
+ if local
+ uri = URI.parse(url)
+ url = "#{uri.request_target}host/#{uri.host}/"
+ end
- # Proxy URLs for video playback on invidious.
- # Other API clients can get the original URLs by omiting `local=true`.
- manifest = response.body.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
- url = baseurl.lchop("").rchop(" ")
- url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
"#{url} "
end
return manifest
end
- # Ditto, only proxify URLs if `local=true` is used
+ adaptive_fmts = video.adaptive_fmts
+
if local
- video.adaptive_fmts.each do |fmt|
- fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
+ adaptive_fmts.each do |fmt|
+ fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target)
end
end
@@ -68,23 +66,17 @@ module Invidious::Routes::API::Manifest
# OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
- audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any
- lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und"
- is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0
- displayname = audio_track["displayName"]?.try &.as_s || "Unknown"
- bitrate = fmt["bitrate"]
-
# Different representations of the same audio should be groupped into one AdaptationSet.
# However, most players don't support auto quality switching, so we have to trick them
# into providing a quality selector.
# See https://github.com/iv-org/invidious/issues/3074 for more details.
- xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do
+ xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
- xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate")
+ xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate")
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
@@ -181,9 +173,8 @@ module Invidious::Routes::API::Manifest
manifest = response.body
if local
- manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match|
- uri = URI.parse(match)
- path = uri.path
+ manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
+ path = URI.parse(match).path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
@@ -212,7 +203,7 @@ module Invidious::Routes::API::Manifest
raw_params["fvip"] = fvip["fvip"]
end
- raw_params["host"] = uri.host.not_nil!
+ raw_params["local"] = "true"
"#{HOST_URL}/videoplayback?#{raw_params}"
end
diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr
index a35d2f2b..1f5ad8ef 100644
--- a/src/invidious/routes/api/v1/authenticated.cr
+++ b/src/invidious/routes/api/v1/authenticated.cr
@@ -31,88 +31,6 @@ module Invidious::Routes::API::V1::Authenticated
env.response.status_code = 204
end
- def self.export_invidious(env)
- env.response.content_type = "application/json"
- user = env.get("user").as(User)
-
- return Invidious::User::Export.to_invidious(user)
- end
-
- def self.import_invidious(env)
- user = env.get("user").as(User)
-
- begin
- if body = env.request.body
- body = env.request.body.not_nil!.gets_to_end
- else
- body = "{}"
- end
- Invidious::User::Import.from_invidious(user, body)
- rescue
- end
-
- env.response.status_code = 204
- end
-
- def self.get_history(env)
- env.response.content_type = "application/json"
- user = env.get("user").as(User)
-
- page = env.params.query["page"]?.try &.to_i?.try &.clamp(0, Int32::MAX)
- page ||= 1
-
- max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
- max_results ||= user.preferences.max_results
- max_results ||= CONFIG.default_user_preferences.max_results
-
- start_index = (page - 1) * max_results
- if user.watched[start_index]?
- watched = user.watched.reverse[start_index, max_results]
- end
- watched ||= [] of String
-
- return watched.to_json
- end
-
- def self.mark_watched(env)
- user = env.get("user").as(User)
-
- if !user.preferences.watch_history
- return error_json(409, "Watch history is disabled in preferences.")
- end
-
- id = env.params.url["id"]
- if !id.match(/^[a-zA-Z0-9_-]{11}$/)
- return error_json(400, "Invalid video id.")
- end
-
- Invidious::Database::Users.mark_watched(user, id)
- env.response.status_code = 204
- end
-
- def self.mark_unwatched(env)
- user = env.get("user").as(User)
-
- if !user.preferences.watch_history
- return error_json(409, "Watch history is disabled in preferences.")
- end
-
- id = env.params.url["id"]
- if !id.match(/^[a-zA-Z0-9_-]{11}$/)
- return error_json(400, "Invalid video id.")
- end
-
- Invidious::Database::Users.mark_unwatched(user, id)
- env.response.status_code = 204
- end
-
- def self.clear_history(env)
- user = env.get("user").as(User)
-
- Invidious::Database::Users.clear_watch_history(user)
- env.response.status_code = 204
- end
-
def self.feed(env)
env.response.content_type = "application/json"
@@ -178,6 +96,10 @@ module Invidious::Routes::API::V1::Authenticated
Invidious::Database::Users.subscribe_channel(user, ucid)
end
+ # For Google accounts, access tokens don't have enough information to
+ # make a request on the user's behalf, which is why we don't sync with
+ # YouTube.
+
env.response.status_code = 204
end
@@ -304,8 +226,8 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(403, "Invalid user")
end
- if playlist.index.size >= CONFIG.playlist_length_limit
- return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
+ if playlist.index.size >= 500
+ return error_json(400, "Playlist cannot have more than 500 videos")
end
video_id = env.params.json["videoId"].try &.as(String)
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index a940ee68..6b81c546 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -1,7 +1,13 @@
module Invidious::Routes::API::V1::Channels
- # Macro to avoid duplicating some code below
- # This sets the `channel` variable, or handles Exceptions.
- private macro get_channel
+ def self.home(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ sort_by = env.params.query["sort_by"]?.try &.downcase
+ sort_by ||= "newest"
+
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
@@ -12,33 +18,14 @@ module Invidious::Routes::API::V1::Channels
rescue ex
return error_json(500, ex)
end
- end
- def self.home(env)
- locale = env.get("preferences").as(Preferences).locale
- ucid = env.params.url["ucid"]
-
- env.response.content_type = "application/json"
-
- # Use the private macro defined above.
- channel = nil # Make the compiler happy
- get_channel()
-
- # Retrieve "sort by" setting from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
-
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
+ page = 1
+ if channel.auto_generated
+ videos = [] of SearchVideo
+ count = 0
else
begin
- videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
+ count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
rescue ex
return error_json(500, ex)
end
@@ -95,15 +82,11 @@ module Invidious::Routes::API::V1::Channels
json.field "joined", channel.joined.to_unix
json.field "autoGenerated", channel.auto_generated
- json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html
json.field "allowedRegions", channel.allowed_regions
- json.field "tabs", channel.tabs
- json.field "tags", channel.tags
- json.field "authorVerified", channel.verified
json.field "latestVideos" do
json.array do
@@ -117,13 +100,31 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
- related_channels, _ = fetch_related_channels(channel)
+ related_channels = fetch_related_channels(channel)
rescue ex
- related_channels = [] of SearchChannel
+ related_channels = [] of AboutRelatedChannel
end
related_channels.each do |related_channel|
- related_channel.to_json(locale, json)
+ json.object do
+ json.field "author", related_channel.author
+ json.field "authorId", related_channel.ucid
+ json.field "authorUrl", related_channel.author_url
+
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+ end
end
end
end # relatedChannels
@@ -133,147 +134,61 @@ module Invidious::Routes::API::V1::Channels
end
def self.latest(env)
- # Remove parameters that could affect this endpoint's behavior
- env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
- env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
+ locale = env.get("preferences").as(Preferences).locale
- return self.videos(env)
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+
+ begin
+ videos = get_latest_videos(ucid)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ JSON.build do |json|
+ json.array do
+ videos.each do |video|
+ video.to_json(locale, json)
+ end
+ end
+ end
end
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
- ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
- # Use the private macro defined above.
- channel = nil # Make the compiler happy
- get_channel()
-
- # Retrieve some URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- continuation = env.params.query["continuation"]?
-
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_60_videos(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
- end
-
- return JSON.build do |json|
- json.object do
- json.field "videos" do
- json.array do
- videos.each &.to_json(locale, json)
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
- def self.shorts(env)
- locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
+ page = env.params.query["page"]?.try &.to_i?
+ page ||= 1
+ sort_by = env.params.query["sort"]?.try &.downcase
+ sort_by ||= env.params.query["sort_by"]?.try &.downcase
+ sort_by ||= "newest"
- env.response.content_type = "application/json"
-
- # Use the private macro defined above.
- channel = nil # Make the compiler happy
- get_channel()
-
- # Retrieve continuation from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- continuation = env.params.query["continuation"]?
-
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_shorts(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ channel = get_about_info(ucid, locale)
+ rescue ex : ChannelRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
+ return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
end
- return JSON.build do |json|
- json.object do
- json.field "videos" do
- json.array do
- videos.each &.to_json(locale, json)
- end
+ begin
+ count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ JSON.build do |json|
+ json.array do
+ videos.each do |video|
+ video.to_json(locale, json)
end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
- def self.streams(env)
- locale = env.get("preferences").as(Preferences).locale
- ucid = env.params.url["ucid"]
-
- env.response.content_type = "application/json"
-
- # Use the private macro defined above.
- channel = nil # Make the compiler happy
- get_channel()
-
- # Retrieve continuation from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- continuation = env.params.query["continuation"]?
-
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_60_livestreams(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
- end
-
- return JSON.build do |json|
- json.object do
- json.field "videos" do
- json.array do
- videos.each &.to_json(locale, json)
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
end
end
end
@@ -289,40 +204,18 @@ module Invidious::Routes::API::V1::Channels
env.params.query["sort_by"]?.try &.downcase ||
"last"
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
-
- JSON.build do |json|
- json.object do
- json.field "playlists" do
- json.array do
- items.each do |item|
- item.to_json(locale, json) if item.is_a?(SearchPlaylist)
- end
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
+ begin
+ channel = get_about_info(ucid, locale)
+ rescue ex : ChannelRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
+ return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
end
- end
- def self.podcasts(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- continuation = env.params.query["continuation"]?
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_podcasts(channel.ucid, channel.author, continuation)
+ items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
JSON.build do |json|
json.object do
@@ -334,65 +227,7 @@ module Invidious::Routes::API::V1::Channels
end
end
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
- def self.releases(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- continuation = env.params.query["continuation"]?
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_releases(channel.ucid, channel.author, continuation)
-
- JSON.build do |json|
- json.object do
- json.field "playlists" do
- json.array do
- items.each do |item|
- item.to_json(locale, json) if item.is_a?(SearchPlaylist)
- end
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
- def self.courses(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- continuation = env.params.query["continuation"]?
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation)
-
- JSON.build do |json|
- json.object do
- json.field "playlists" do
- json.array do
- items.each do |item|
- item.to_json(locale, json) if item.is_a?(SearchPlaylist)
- end
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
+ json.field "continuation", continuation
end
end
end
@@ -420,90 +255,6 @@ module Invidious::Routes::API::V1::Channels
end
end
- def self.post(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
- id = env.params.url["id"].to_s
- ucid = env.params.query["ucid"]?
-
- thin_mode = env.params.query["thin_mode"]?
- thin_mode = thin_mode == "true"
-
- format = env.params.query["format"]?
- format ||= "json"
-
- if ucid.nil?
- response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
- return error_json(400, "Invalid post ID") if response["error"]?
- ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
- else
- ucid = ucid.to_s
- end
-
- begin
- fetch_channel_community_post(ucid, id, locale, format, thin_mode)
- rescue ex
- return error_json(500, ex)
- end
- end
-
- def self.post_comments(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- id = env.params.url["id"]
-
- thin_mode = env.params.query["thin_mode"]?
- thin_mode = thin_mode == "true"
-
- format = env.params.query["format"]?
- format ||= "json"
-
- continuation = env.params.query["continuation"]?
-
- case continuation
- when nil, ""
- ucid = env.params.query["ucid"]
- comments = Comments.fetch_community_post_comments(ucid, id)
- else
- comments = YoutubeAPI.browse(continuation: continuation)
- end
- return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true)
- end
-
- def self.channels(env)
- locale = env.get("preferences").as(Preferences).locale
- ucid = env.params.url["ucid"]
-
- env.response.content_type = "application/json"
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- continuation = env.params.query["continuation"]?
-
- begin
- items, next_continuation = fetch_related_channels(channel, continuation)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.object do
- json.field "relatedChannels" do
- json.array do
- items.each &.to_json(locale, json)
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr
index fea2993c..41865f34 100644
--- a/src/invidious/routes/api/v1/feeds.cr
+++ b/src/invidious/routes/api/v1/feeds.cr
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds
if !CONFIG.popular_enabled
error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
- haltf env, 403, error_message
+ haltf env, 400, error_message
end
JSON.build do |json|
diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr
index 4f5b58da..844fedb8 100644
--- a/src/invidious/routes/api/v1/misc.cr
+++ b/src/invidious/routes/api/v1/misc.cr
@@ -6,22 +6,6 @@ module Invidious::Routes::API::V1::Misc
if !CONFIG.statistics_enabled
return {"software" => SOFTWARE}.to_json
else
- # Calculate playback success rate
- if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
- tracker = tracker.as(Hash(String, Int64 | Float64))
-
- if !tracker.empty?
- total_requests = tracker["totalRequests"]
- success_count = tracker["successfulRequests"]
-
- if total_requests.zero?
- tracker["ratio"] = 1_i64
- else
- tracker["ratio"] = (success_count / (total_requests)).round(2)
- end
- end
- end
-
return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
end
@@ -42,9 +26,6 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
- listen_param = env.params.query["listen"]?
- listen = (listen_param == "true" || listen_param == "1")
-
if plid.starts_with? "RD"
return env.redirect "/api/v1/mixes/#{plid}"
end
@@ -77,9 +58,7 @@ module Invidious::Routes::API::V1::Misc
response = playlist.to_json(offset, video_id: video_id)
json_response = JSON.parse(response)
- if json_response["videos"].as_a.empty?
- json_response = JSON.parse(response)
- elsif json_response["videos"].as_a[0]["index"] != offset
+ if json_response["videos"].as_a[0]["index"] != offset
offset = json_response["videos"].as_a[0]["index"].as_i
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback)
@@ -88,7 +67,7 @@ module Invidious::Routes::API::V1::Misc
end
if format == "html"
- playlist_html = template_playlist(json_response, listen)
+ playlist_html = template_playlist(json_response)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = {
@@ -114,9 +93,6 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
- listen_param = env.params.query["listen"]?
- listen = (listen_param == "true" || listen_param == "1")
-
begin
mix = fetch_mix(rdid, continuation, locale: locale)
@@ -147,7 +123,9 @@ module Invidious::Routes::API::V1::Misc
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "videoThumbnails" do
- Invidious::JSONify::APIv1.thumbnails(json, video.id)
+ json.array do
+ generate_thumbnails(json, video.id)
+ end
end
json.field "index", video.index
@@ -161,7 +139,7 @@ module Invidious::Routes::API::V1::Misc
if format == "html"
response = JSON.parse(response)
- playlist_html = template_mix(response, listen)
+ playlist_html = template_mix(response)
next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"]
response = {
@@ -172,36 +150,4 @@ module Invidious::Routes::API::V1::Misc
response
end
-
- # resolve channel and clip urls, return the UCID
- def self.resolve_url(env)
- env.response.content_type = "application/json"
- url = env.params.query["url"]?
-
- return error_json(400, "Missing URL to resolve") if !url
-
- begin
- resolved_url = YoutubeAPI.resolve_url(url.as(String))
- endpoint = resolved_url["endpoint"]
- page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
- if page_type == "WEB_PAGE_TYPE_UNKNOWN"
- return error_json(400, "Unknown url")
- end
-
- sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
- params = sub_endpoint.try &.dig?("params")
- rescue ex
- return error_json(500, ex)
- end
- JSON.build do |json|
- json.object do
- json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
- json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
- json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
- json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
- json.field "params", params.try &.as_s
- json.field "pageType", page_type
- end
- end
- end
end
diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 59a30745..21451d33 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -31,13 +31,12 @@ module Invidious::Routes::API::V1::Search
query = env.params.query["q"]? || ""
begin
- client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
- url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
+ client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
+ url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt"
response = client.get(url).body
- client.close
- body = JSON.parse(response[19..-2]).as_a
+ body = JSON.parse(response[5..-1]).as_a
suggestions = body[1].as_a[0..-2]
JSON.build do |json|
@@ -56,32 +55,4 @@ module Invidious::Routes::API::V1::Search
return error_json(500, ex)
end
end
-
- def self.hashtag(env)
- hashtag = env.params.url["hashtag"]
-
- page = env.params.query["page"]?.try &.to_i? || 1
-
- locale = env.get("preferences").as(Preferences).locale
- region = env.params.query["region"]?
- env.response.content_type = "application/json"
-
- begin
- results = Invidious::Hashtag.fetch(hashtag, page, region)
- rescue ex
- return error_json(400, ex)
- end
-
- JSON.build do |json|
- json.object do
- json.field "results" do
- json.array do
- results.each do |item|
- item.to_json(locale, json)
- end
- end
- end
- end
- end
- end
end
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index 6a3eb8ae..1b7b4fa7 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -1,5 +1,3 @@
-require "html"
-
module Invidious::Routes::API::V1::Videos
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
@@ -8,19 +6,19 @@ module Invidious::Routes::API::V1::Videos
id = env.params.url["id"]
region = env.params.query["region"]?
- proxy = {"1", "true"}.any? &.== env.params.query["local"]?
begin
video = get_video(id, region: region)
+ rescue ex : VideoRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
+ return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
- return JSON.build do |json|
- Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
- end
+ video.to_json(locale, nil)
end
def self.captions(env)
@@ -43,6 +41,9 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
+ rescue ex : VideoRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
+ return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
haltf env, 404
rescue ex
@@ -89,77 +90,61 @@ module Invidious::Routes::API::V1::Videos
caption = caption[0]
end
- if CONFIG.use_innertube_for_captions
- params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
+ url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
- transcript = Invidious::Videos::Transcript.from_raw(
- YoutubeAPI.get_transcript(params),
- caption.language_code,
- caption.auto_generated
- )
+ # Auto-generated captions often have cues that aren't aligned properly with the video,
+ # as well as some other markup that makes it cumbersome, so we try to fix that here
+ if caption.name.includes? "auto-generated"
+ caption_xml = YT_POOL.client &.get(url).body
+ caption_xml = XML.parse(caption_xml)
- webvtt = transcript.to_vtt
- else
- # Timedtext API handling
- url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
+ webvtt = String.build do |str|
+ str << <<-END_VTT
+ WEBVTT
+ Kind: captions
+ Language: #{tlang || caption.language_code}
- # Auto-generated captions often have cues that aren't aligned properly with the video,
- # as well as some other markup that makes it cumbersome, so we try to fix that here
- if caption.name.includes? "auto-generated"
- caption_xml = YT_POOL.client &.get(url).body
- settings_field = {
- "Kind" => "captions",
- "Language" => "#{tlang || caption.language_code}",
- }
+ END_VTT
- if caption_xml.starts_with?(" i + 1
- end_time = caption_nodes[i + 1]["start"].to_f.seconds
- else
- end_time = start_time + duration
- end
-
- text = HTML.unescape(node.content)
- text = text.gsub(//, "")
- text = text.gsub(/<\/font>/, "")
- if md = text.match(/(?.*) : (?.*)/)
- text = "#{md["text"]} "
- end
-
- builder.cue(start_time, end_time, text)
- end
+ if caption_nodes.size > i + 1
+ end_time = caption_nodes[i + 1]["start"].to_f.seconds
+ else
+ end_time = start_time + duration
end
- end
- else
- uri = URI.parse(url)
- query_params = uri.query_params
- query_params["fmt"] = "vtt"
- uri.query_params = query_params
- webvtt = YT_POOL.client &.get(uri.request_target).body
- if webvtt.starts_with?(" [0-9:.]{12}).+/, "\\1")
+ start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
+ end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
+
+ text = HTML.unescape(node.content)
+ text = text.gsub(//, "")
+ text = text.gsub(/<\/font>/, "")
+ if md = text.match(/(?.*) : (?.*)/)
+ text = "#{md["text"]} "
+ end
+
+ str << <<-END_CUE
+ #{start_time} --> #{end_time}
+ #{text}
+
+
+ END_CUE
end
end
+ else
+ # Some captions have "align:[start/end]" and "position:[num]%"
+ # attributes. Those are causing issues with VideoJS, which is unable
+ # to properly align the captions on the video, so we remove them.
+ #
+ # See: https://github.com/iv-org/invidious/issues/2391
+ webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
+ .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end
if title = env.params.query["title"]?
@@ -183,20 +168,24 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
+ rescue ex : VideoRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
+ return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
haltf env, 404
rescue ex
haltf env, 500
end
- width = env.params.query["width"]?.try &.to_i
- height = env.params.query["height"]?.try &.to_i
+ storyboards = video.storyboards
+ width = env.params.query["width"]?
+ height = env.params.query["height"]?
if !width && !height
response = JSON.build do |json|
json.object do
json.field "storyboards" do
- Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards)
+ generate_storyboards(json, id, storyboards)
end
end
end
@@ -206,48 +195,43 @@ module Invidious::Routes::API::V1::Videos
env.response.content_type = "text/vtt"
- # Select a storyboard matching the user's provided width/height
- storyboard = video.storyboards.select { |x| x.width == width || x.height == height }
- haltf env, 404 if storyboard.empty?
+ storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" }
- # Alias variable, to make the code below esaier to read
- sb = storyboard[0]
+ if storyboard.empty?
+ haltf env, 404
+ else
+ storyboard = storyboard[0]
+ end
- # Some base URL segments that we'll use to craft the final URLs
- work_url = sb.proxied_url.dup
- template_path = sb.proxied_url.path
+ String.build do |str|
+ str << <<-END_VTT
+ WEBVTT
+ END_VTT
- # Initialize cue timing variables
- # NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap
- # (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000)
- time_delta = sb.interval.milliseconds
- start_time = 0.milliseconds
- end_time = time_delta
+ start_time = 0.milliseconds
+ end_time = storyboard[:interval].milliseconds
- # Build a VTT file for VideoJS-vtt plugin
- vtt_file = WebVTT.build do |vtt|
- sb.images_count.times do |i|
- # Replace the variable component part of the path
- work_url.path = template_path.sub("$M", i)
+ storyboard[:storyboard_count].times do |i|
+ url = storyboard[:url]
+ authority = /(i\d?).ytimg.com/.match(url).not_nil![1]?
+ url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "")
+ url = "#{HOST_URL}/sb/#{authority}/#{url}"
- sb.rows.times do |j|
- sb.columns.times do |k|
- # The URL fragment represents the offset of the thumbnail inside the storyboard image
- work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}"
+ storyboard[:storyboard_height].times do |j|
+ storyboard[:storyboard_width].times do |k|
+ str << <<-END_CUE
+ #{start_time}.000 --> #{end_time}.000
+ #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}
- vtt.cue(start_time, end_time, work_url.to_s)
- start_time += time_delta
- end_time += time_delta
+ END_CUE
+
+ start_time += storyboard[:interval].milliseconds
+ end_time += storyboard[:interval].milliseconds
end
end
end
end
-
- # videojs-vtt-thumbnails is not compliant to the VTT specification, it
- # doesn't unescape the HTML entities, so we have to do it here:
- # TODO: remove this when we migrate to VideoJS 8
- return HTML.unescape(vtt_file)
end
def self.annotations(env)
@@ -268,7 +252,7 @@ module Invidious::Routes::API::V1::Videos
if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
annotations = cached_annotation.annotations
else
- index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
+ index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0')
# IA doesn't handle leading hyphens,
# so we use https://archive.org/details/youtubeannotations_64
@@ -345,7 +329,7 @@ module Invidious::Routes::API::V1::Videos
sort_by ||= "top"
begin
- comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
+ comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
@@ -357,7 +341,7 @@ module Invidious::Routes::API::V1::Videos
sort_by ||= "confidence"
begin
- comments, reddit_thread = Comments.fetch_reddit(id, sort_by: sort_by)
+ comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by)
rescue ex
comments = nil
reddit_thread = nil
@@ -373,9 +357,9 @@ module Invidious::Routes::API::V1::Videos
return reddit_thread.to_json
else
- content_html = Frontend::Comments.template_reddit(comments, locale)
- content_html = Comments.fill_links(content_html, "https", "www.reddit.com")
- content_html = Comments.replace_links(content_html)
+ content_html = template_reddit_comments(comments, locale)
+ content_html = fill_links(content_html, "https", "www.reddit.com")
+ content_html = replace_links(content_html)
response = {
"title" => reddit_thread.title,
"permalink" => reddit_thread.permalink,
@@ -386,133 +370,4 @@ module Invidious::Routes::API::V1::Videos
end
end
end
-
- def self.clips(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- clip_id = env.params.url["id"]
- region = env.params.query["region"]?
- proxy = {"1", "true"}.any? &.== env.params.query["local"]?
-
- response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
- return error_json(400, "Invalid clip ID") if response["error"]?
-
- video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
- return error_json(400, "Invalid clip ID") if video_id.nil?
-
- start_time = nil
- end_time = nil
- clip_title = nil
-
- if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
- start_time, end_time, clip_title = parse_clip_parameters(params)
- end
-
- begin
- video = get_video(video_id, region: region)
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
-
- return JSON.build do |json|
- json.object do
- json.field "startTime", start_time
- json.field "endTime", end_time
- json.field "clipTitle", clip_title
- json.field "video" do
- Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
- end
- end
- end
- end
-
- # Fetches transcripts from YouTube
- #
- # Use the `lang` and `autogen` query parameter to select which transcript to fetch
- # Request without any URL parameters to see all the available transcripts.
- def self.transcripts(env)
- env.response.content_type = "application/json"
-
- id = env.params.url["id"]
- lang = env.params.query["lang"]?
- label = env.params.query["label"]?
- auto_generated = env.params.query["autogen"]? ? true : false
-
- # Return all available transcript options when none is given
- if !label && !lang
- begin
- video = get_video(id)
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
-
- response = JSON.build do |json|
- # The amount of transcripts available to fetch is the
- # same as the amount of captions available.
- available_transcripts = video.captions
-
- json.object do
- json.field "transcripts" do
- json.array do
- available_transcripts.each do |transcript|
- json.object do
- json.field "label", transcript.name
- json.field "languageCode", transcript.language_code
- json.field "autoGenerated", transcript.auto_generated
-
- if transcript.auto_generated
- json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}&autogen"
- else
- json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}"
- end
- end
- end
- end
- end
- end
- end
-
- return response
- end
-
- # If lang is not given then we attempt to fetch
- # the transcript through the given label
- if lang.nil?
- begin
- video = get_video(id)
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
-
- target_transcript = video.captions.select(&.name.== label)
- if target_transcript.empty?
- return error_json(404, NotFoundException.new("Requested transcript does not exist"))
- else
- target_transcript = target_transcript[0]
- lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated
- end
- end
-
- params = Invidious::Videos::Transcript.generate_param(id, lang, auto_generated)
-
- begin
- transcript = Invidious::Videos::Transcript.from_raw(
- YoutubeAPI.get_transcript(params), lang, auto_generated
- )
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
-
- return transcript.to_json
- end
end
diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr
index b5269668..8e2a253f 100644
--- a/src/invidious/routes/before_all.cr
+++ b/src/invidious/routes/before_all.cr
@@ -20,9 +20,17 @@ module Invidious::Routes::BeforeAll
env.response.headers["X-XSS-Protection"] = "1; mode=block"
env.response.headers["X-Content-Type-Options"] = "nosniff"
+ # Allow media resources to be loaded from google servers
+ # TODO: check if *.youtube.com can be removed
+ if CONFIG.disabled?("local") || !preferences.local
+ extra_media_csp = " https://*.googlevideo.com:443 https://*.youtube.com:443"
+ else
+ extra_media_csp = ""
+ end
+
# Only allow the pages at /embed/* to be embedded
if env.request.resource.starts_with?("/embed")
- frame_ancestors = "'self' file: http: https:"
+ frame_ancestors = "'self' http: https:"
else
frame_ancestors = "'none'"
end
@@ -37,7 +45,7 @@ module Invidious::Routes::BeforeAll
"font-src 'self' data:",
"connect-src 'self'",
"manifest-src 'self'",
- "media-src 'self' blob:",
+ "media-src 'self' blob:" + extra_media_csp,
"child-src 'self' blob:",
"frame-src 'self'",
"frame-ancestors " + frame_ancestors,
@@ -72,23 +80,49 @@ module Invidious::Routes::BeforeAll
raise "Cannot use token as SID"
end
- if email = Database::SessionIDs.select_email(sid)
- user = Database::Users.select!(email: email)
- csrf_token = generate_response(sid, {
- ":authorize_token",
- ":playlist_ajax",
- ":signout",
- ":subscription_ajax",
- ":token_ajax",
- ":watch_ajax",
- }, HMAC_KEY, 1.week)
+ # Invidious users only have SID
+ if !env.request.cookies.has_key? "SSID"
+ if email = Invidious::Database::SessionIDs.select_email(sid)
+ user = Invidious::Database::Users.select!(email: email)
+ csrf_token = generate_response(sid, {
+ ":authorize_token",
+ ":playlist_ajax",
+ ":signout",
+ ":subscription_ajax",
+ ":token_ajax",
+ ":watch_ajax",
+ }, HMAC_KEY, 1.week)
- preferences = user.preferences
- env.set "preferences", preferences
+ preferences = user.preferences
+ env.set "preferences", preferences
- env.set "sid", sid
- env.set "csrf_token", csrf_token
- env.set "user", user
+ env.set "sid", sid
+ env.set "csrf_token", csrf_token
+ env.set "user", user
+ end
+ else
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env.request.headers["Cookie"]
+
+ begin
+ user, sid = get_user(sid, headers, false)
+ csrf_token = generate_response(sid, {
+ ":authorize_token",
+ ":playlist_ajax",
+ ":signout",
+ ":subscription_ajax",
+ ":token_ajax",
+ ":watch_ajax",
+ }, HMAC_KEY, 1.week)
+
+ preferences = user.preferences
+ env.set "preferences", preferences
+
+ env.set "sid", sid
+ env.set "csrf_token", csrf_token
+ env.set "user", user
+ rescue ex
+ end
end
end
@@ -102,21 +136,6 @@ module Invidious::Routes::BeforeAll
preferences.locale = locale
env.set "preferences", preferences
- # Allow media resources to be loaded from google servers
- # TODO: check if *.youtube.com can be removed
- #
- # `!preferences.local` has to be checked after setting and
- # reading `preferences` from the "PREFS" cookie and
- # saved user preferences from the database, otherwise
- # `https://*.googlevideo.com:443 https://*.youtube.com:443`
- # will not be set in the CSP header if
- # `default_user_preferences.local` is set to true on the
- # configuration file, causing preference “Proxy Videos”
- # not to work while having it disabled and using medium quality.
- if CONFIG.disabled?("local") || !preferences.local
- env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"].gsub("media-src", "media-src https://*.googlevideo.com:443 https://*.youtube.com:443")
- end
-
current_page = env.request.path
if env.request.query
query = HTTP::Params.parse(env.request.query.not_nil!)
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 508aa3e4..c6e02cbd 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -1,32 +1,27 @@
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Channels
- # Redirection for unsupported routes ("tabs")
- def self.redirect_home(env)
- ucid = env.params.url["ucid"]
- return env.redirect "/channel/#{URI.encode_www_form(ucid)}"
- end
-
def self.home(env)
self.videos(env)
end
def self.videos(env)
data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
+ if !data.is_a?(Tuple)
+ return data
+ end
locale, user, subscriptions, continuation, ucid, channel = data
+ page = env.params.query["page"]?.try &.to_i?
+ page ||= 1
+
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
- sort_by ||= "last"
sort_options = {"last", "oldest", "newest"}
+ sort_by ||= "last"
- items, next_continuation = fetch_channel_playlists(
- channel.ucid, channel.author, continuation, sort_by
- )
-
+ items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@@ -34,216 +29,57 @@ module Invidious::Routes::Channels
item.author
end
end
- items = items.select(SearchPlaylist)
+ items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
items.each(&.author = "")
else
- # Fetch items and continuation token
- if channel.is_age_gated
- sort_by = ""
- sort_options = [] of String
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- items = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- items = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- sort_by ||= "newest"
- sort_options = {"newest", "oldest", "popular"}
-
- items, next_continuation = Channel::Tabs.get_60_videos(
- channel, continuation: continuation, sort_by: sort_by
- )
- end
- end
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
- templated "channel"
- end
-
- def self.shorts(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- if !channel.tabs.includes? "shorts"
- return env.redirect "/channel/#{channel.ucid}"
- end
-
- if channel.is_age_gated
- sort_by = ""
- sort_options = [] of String
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
- items = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- items = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
sort_options = {"newest", "oldest", "popular"}
+ sort_by ||= "newest"
- # Fetch items and continuation token
- items, next_continuation = Channel::Tabs.get_shorts(
- channel, continuation: continuation, sort_by: sort_by
- )
+ count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
end
- selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
- templated "channel"
- end
-
- def self.streams(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- if !channel.tabs.includes? "streams"
- return env.redirect "/channel/#{channel.ucid}"
- end
-
- if channel.is_age_gated
- sort_by = ""
- sort_options = [] of String
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
- items = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- items = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- sort_options = {"newest", "oldest", "popular"}
-
- # Fetch items and continuation token
- items, next_continuation = Channel::Tabs.get_60_livestreams(
- channel, continuation: continuation, sort_by: sort_by
- )
- end
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
def self.playlists(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- sort_options = {"last", "oldest", "newest"}
- sort_by = env.params.query["sort_by"]?.try &.downcase
-
- if channel.auto_generated
- return env.redirect "/channel/#{channel.ucid}"
- end
-
- items, next_continuation = fetch_channel_playlists(
- channel.ucid, channel.author, continuation, (sort_by || "last")
- )
-
- items = items.select(SearchPlaylist)
- items.each(&.author = "")
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
- templated "channel"
- end
-
- def self.podcasts(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- sort_by = ""
- sort_options = [] of String
-
- items, next_continuation = fetch_channel_podcasts(
- channel.ucid, channel.author, continuation
- )
-
- items = items.select(SearchPlaylist)
- items.each(&.author = "")
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Podcasts
- templated "channel"
- end
-
- def self.releases(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- sort_by = ""
- sort_options = [] of String
-
- items, next_continuation = fetch_channel_releases(
- channel.ucid, channel.author, continuation
- )
-
- items = items.select(SearchPlaylist)
- items.each(&.author = "")
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Releases
- templated "channel"
- end
-
- def self.courses(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- sort_by = ""
- sort_options = [] of String
-
- items, next_continuation = fetch_channel_courses(
- channel.ucid, channel.author, continuation
- )
-
- items = items.select(SearchPlaylist)
- items.each(&.author = "")
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Courses
- templated "channel"
- end
-
- def self.community(env)
- return env.redirect env.request.path.sub("posts", "community") if env.request.path.split("/").last == "posts"
-
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
locale, user, subscriptions, continuation, ucid, channel = data
- # redirect to post page
- if lb = env.params.query["lb"]?
- env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}"
+ sort_options = {"last", "oldest", "newest"}
+ sort_by = env.params.query["sort_by"]?.try &.downcase
+ sort_by ||= "last"
+
+ if channel.auto_generated
+ return env.redirect "/channel/#{channel.ucid}"
end
+ items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
+ items.each(&.author = "")
+
+ templated "playlists"
+ end
+
+ def self.community(env)
+ data = self.fetch_basic_information(env)
+ if !data.is_a?(Tuple)
+ return data
+ end
+ locale, user, subscriptions, continuation, ucid, channel = data
+
thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
+ # sort_by = env.params.query["sort_by"]?.try &.downcase
- if !channel.tabs.includes? "community" && "posts"
+ if !channel.tabs.includes? "community"
return env.redirect "/channel/#{channel.ucid}"
end
- # TODO: support sort options for community posts
- sort_by = ""
- sort_options = [] of String
-
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
@@ -259,64 +95,6 @@ module Invidious::Routes::Channels
templated "community"
end
- def self.post(env)
- # /post/{postId}
- id = env.params.url["id"]
- ucid = env.params.query["ucid"]?
-
- prefs = env.get("preferences").as(Preferences)
-
- locale = prefs.locale
-
- thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode
- thin_mode = thin_mode == "true"
-
- nojs = env.params.query["nojs"]?
-
- nojs ||= "0"
- nojs = nojs == "1"
-
- if !ucid.nil?
- ucid = ucid.to_s
- post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
- else
- # resolve the url to get the author's UCID
- response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
- return error_template(400, "Invalid post ID") if response["error"]?
-
- ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
- post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
- end
-
- post_response = JSON.parse(post_response)
-
- if nojs
- comments = Comments.fetch_community_post_comments(ucid, id)
- comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"]
- end
- templated "post"
- end
-
- def self.channels(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- if channel.auto_generated
- return env.redirect "/channel/#{channel.ucid}"
- end
-
- items, next_continuation = fetch_related_channels(channel, continuation)
-
- # Featured/related channels can't be sorted
- sort_options = [] of String
- sort_by = nil
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
- templated "channel"
- end
-
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
@@ -327,12 +105,6 @@ module Invidious::Routes::Channels
env.redirect "/channel/#{ucid}"
end
- private KNOWN_TABS = {
- "home", "videos", "shorts", "streams", "podcasts",
- "releases", "courses", "playlists", "community", "channels", "about",
- "posts",
- }
-
# Redirects brand url channels to a normal /channel/:ucid route
def self.brand_redirect(env)
locale = env.get("preferences").as(Preferences).locale
@@ -343,10 +115,7 @@ module Invidious::Routes::Channels
yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"]))
# Retrieves URL params that only Invidious uses
- invidious_url_params = env.params.query.dup
- invidious_url_params.delete_all("a")
- invidious_url_params.delete_all("u")
- invidious_url_params.delete_all("user")
+ invidious_url_params = URI::Params.encode(env.params.query.to_h.select!(["a", "u", "user"]))
begin
resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}")
@@ -355,17 +124,14 @@ module Invidious::Routes::Channels
return error_template(404, translate(locale, "This channel does not exist."))
end
- selected_tab = env.params.url["tab"]?
-
- if KNOWN_TABS.includes? selected_tab
+ selected_tab = env.request.path.split("/")[-1]
+ if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"
end
- url += "?#{invidious_url_params}" if !invidious_url_params.empty?
-
- return env.redirect url
+ env.redirect url
end
# Handles redirects for the /profile endpoint
@@ -440,7 +206,6 @@ module Invidious::Routes::Channels
return error_template(500, ex)
end
- env.set "search", "channel:#{ucid} "
return {locale, user, subscriptions, continuation, ucid, channel}
end
end
diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr
index 930e4915..84da9993 100644
--- a/src/invidious/routes/embed.cr
+++ b/src/invidious/routes/embed.cr
@@ -2,25 +2,18 @@
module Invidious::Routes::Embed
def self.redirect(env)
- locale = env.get("preferences").as(Preferences).locale
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
- if videos.empty?
- url = "/playlist?list=#{plid}"
- raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
- end
-
- first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{first_playlist_video}?#{env.params.query}"
+ url = "/embed/#{videos[0].id}?#{env.params.query}"
if env.params.query.size > 0
url += "?#{env.params.query}"
@@ -33,7 +26,6 @@ module Invidious::Routes::Embed
end
def self.show(env)
- locale = env.get("preferences").as(Preferences).locale
id = env.params.url["id"]
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
@@ -70,19 +62,13 @@ module Invidious::Routes::Embed
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
- if videos.empty?
- url = "/playlist?list=#{plid}"
- raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
- end
-
- first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{first_playlist_video.id}"
+ url = "/embed/#{videos[0].id}"
elsif video_series
url = "/embed/#{video_series.shift}"
env.params.query["playlist"] = video_series.join(",")
@@ -135,6 +121,8 @@ module Invidious::Routes::Embed
begin
video = get_video(id, region: params.region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
@@ -151,7 +139,7 @@ module Invidious::Routes::Embed
# PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
# end
- if CONFIG.enable_user_notifications && notifications && notifications.includes? id
+ if notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
@@ -161,12 +149,10 @@ module Invidious::Routes::Embed
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
end
- # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -207,14 +193,6 @@ module Invidious::Routes::Embed
return env.redirect url
end
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- env.response.headers["Content-Security-Policy"] =
- env.response.headers["Content-Security-Policy"]
- .gsub("media-src", "media-src #{invidious_companion.public_url}")
- .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
- end
-
rendered "embed"
end
end
diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr
index 1e9ab44e..b138b562 100644
--- a/src/invidious/routes/errors.cr
+++ b/src/invidious/routes/errors.cr
@@ -1,10 +1,5 @@
module Invidious::Routes::ErrorRoutes
def self.error_404(env)
- # Workaround for #3117
- if HOST_URL.empty? && env.request.path.starts_with?("/v1/storyboards/sb")
- return env.redirect "#{env.request.path[15..]}?#{env.params.query}"
- end
-
if md = env.request.path.match(/^\/(?([a-zA-Z0-9_-]{11})|(\w+))$/)
item = md["id"]
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index 070c96eb..b601db94 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -83,6 +83,10 @@ module Invidious::Routes::Feeds
headers = HTTP::Headers.new
headers["Cookie"] = env.request.headers["Cookie"]
+ if !user.password
+ user, sid = get_user(sid, headers)
+ end
+
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
@@ -92,20 +96,14 @@ module Invidious::Routes::Feeds
videos, notifications = get_subscription_feed(user, max_results, page)
- if CONFIG.enable_user_notifications
- # "updated" here is used for delivering new notifications, so if
- # we know a user has looked at their feed e.g. in the past 10 minutes,
- # they've already seen a video posted 20 minutes ago, and don't need
- # to be notified.
- Invidious::Database::Users.clear_notifications(user)
- user.notifications = [] of String
- end
+ # "updated" here is used for delivering new notifications, so if
+ # we know a user has looked at their feed e.g. in the past 10 minutes,
+ # they've already seen a video posted 20 minutes ago, and don't need
+ # to be notified.
+ Invidious::Database::Users.clear_notifications(user)
+ user.notifications = [] of String
env.set "user", user
- # Used for pagination links
- base_url = "/feed/subscriptions"
- base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
-
templated "feeds/subscriptions"
end
@@ -133,91 +131,87 @@ module Invidious::Routes::Feeds
end
watched ||= [] of String
- # Used for pagination links
- base_url = "/feed/history"
- base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
-
templated "feeds/history"
end
# RSS feeds
def self.rss_channel(env)
+ locale = env.get("preferences").as(Preferences).locale
+
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
- if env.params.url["ucid"].matches?(/^[\w-]+$/)
- ucid = env.params.url["ucid"]
- else
- return error_atom(400, InfoException.new("Invalid channel ucid provided."))
- end
+ ucid = env.params.url["ucid"]
params = HTTP::Params.parse(env.params.query["params"]? || "")
- namespaces = {
- "yt" => "http://www.youtube.com/xml/schemas/2015",
- "media" => "http://search.yahoo.com/mrss/",
- "default" => "http://www.w3.org/2005/Atom",
- }
+ begin
+ channel = get_about_info(ucid, locale)
+ rescue ex : ChannelRedirect
+ return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
+ rescue ex : NotFoundException
+ return error_atom(404, ex)
+ rescue ex
+ return error_atom(500, ex)
+ end
- response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
- return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
- rss = XML.parse(response.body)
+ response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
+ rss = XML.parse_html(response.body)
- videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
- video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
- title = entry.xpath_node("default:title", namespaces).not_nil!.content
+ videos = rss.xpath_nodes("//feed/entry").map do |entry|
+ video_id = entry.xpath_node("videoid").not_nil!.content
+ title = entry.xpath_node("title").not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
+ published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
- author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
- video_ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
- description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s
- views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64
+ author = entry.xpath_node("author/name").not_nil!.content
+ ucid = entry.xpath_node("channelid").not_nil!.content
+ description_html = entry.xpath_node("group/description").not_nil!.to_s
+ views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
SearchVideo.new({
title: title,
id: video_id,
author: author,
- ucid: video_ucid,
+ ucid: ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
+ live_now: false,
+ paid: false,
+ premium: false,
premiere_timestamp: nil,
author_verified: false,
- author_thumbnail: nil,
- badges: VideoBadges::None,
})
end
- author = ""
- author = videos[0].author if videos.size > 0
-
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
- xml.element("id") { xml.text "yt:channel:#{ucid}" }
- xml.element("yt:channelId") { xml.text ucid }
- xml.element("title") { xml.text author }
- xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{ucid}")
+ xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
+ xml.element("yt:channelId") { xml.text channel.ucid }
+ xml.element("icon") { xml.text channel.author_thumbnail }
+ xml.element("title") { xml.text channel.author }
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
xml.element("author") do
- xml.element("name") { xml.text author }
- xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
+ xml.element("name") { xml.text channel.author }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
end
xml.element("image") do
- xml.element("url") { xml.text "" }
- xml.element("title") { xml.text author }
+ xml.element("url") { xml.text channel.author_thumbnail }
+ xml.element("title") { xml.text channel.author }
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
end
videos.each do |video|
- video.to_xml(false, params, xml)
+ video.to_xml(channel.auto_generated, params, xml)
end
end
end
@@ -296,13 +290,7 @@ module Invidious::Routes::Feeds
xml.element("name") { xml.text playlist.author }
end
- videos.each do |video|
- if video.is_a? PlaylistVideo
- video.to_xml(xml)
- else
- video.to_xml(env, locale, xml)
- end
- end
+ videos.each &.to_xml(xml)
end
end
else
@@ -311,9 +299,8 @@ module Invidious::Routes::Feeds
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
- return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
-
document = XML.parse(response.body)
+
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
node.attributes.each do |attribute|
case attribute.name
@@ -408,23 +395,22 @@ module Invidious::Routes::Feeds
end
spawn do
- # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow?
- namespaces = {
- "yt" => "http://www.youtube.com/xml/schemas/2015",
- "default" => "http://www.w3.org/2005/Atom",
- }
- rss = XML.parse(body)
- rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
- id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
- author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
+ rss = XML.parse_html(body)
+ rss.xpath_nodes("//feed/entry").each do |entry|
+ id = entry.xpath_node("videoid").not_nil!.content
+ author = entry.xpath_node("author/name").not_nil!.content
+ published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
- begin
- video = get_video(id, force_refresh: true)
- rescue
- next # skip this video since it raised an exception (e.g. it is a scheduled live event)
- end
+ video = get_video(id, force_refresh: true)
+
+ # Deliver notifications to `/api/v1/auth/notifications`
+ payload = {
+ "topic" => video.ucid,
+ "videoId" => video.id,
+ "published" => published.to_unix,
+ }.to_json
+ PG_DB.exec("NOTIFY notifications, E'#{payload}'")
video = ChannelVideo.new({
id: id,
@@ -440,9 +426,7 @@ module Invidious::Routes::Feeds
})
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
- if was_insert
- NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
- end
+ Invidious::Database::Users.add_notification(video) if was_insert
end
end
diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 51d85dfe..594a7869 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -3,7 +3,17 @@ module Invidious::Routes::Images
def self.ggpht(env)
url = env.request.path.lchop("/ggpht")
- headers = HTTP::Headers.new
+ headers = (
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ HTTP::Headers{":authority" => "yt3.ggpht.com"}
+ else
+ HTTP::Headers.new
+ end
+ {% else %}
+ HTTP::Headers.new
+ {% end %}
+ )
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
@@ -11,10 +21,43 @@ module Invidious::Routes::Images
end
end
- begin
- GGPHT_POOL.client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ # We're encapsulating this into a proc in order to easily reuse this
+ # portion of the code for each request block below.
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ env.response.headers.delete("Transfer-Encoding")
+ return
+ end
+
+ proxy_file(response, env)
+ }
+
+ begin
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ YT_POOL.client &.get(url, headers) do |resp|
+ return request_proc.call(resp)
+ end
+ else
+ HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ {% end %}
rescue ex
end
end
@@ -35,17 +78,51 @@ module Invidious::Routes::Images
headers = HTTP::Headers.new
+ {% unless flag?(:disable_quic) %}
+ headers[":authority"] = "#{authority}.ytimg.com"
+ {% end %}
+
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
- begin
- get_ytimg_pool(authority).client &.get(url, headers) do |resp|
- env.response.headers["Connection"] = "close"
- return self.proxy_image(env, resp)
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
end
+
+ env.response.headers["Connection"] = "close"
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
+ begin
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ YT_POOL.client &.get(url, headers) do |resp|
+ return request_proc.call(resp)
+ end
+ else
+ HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ {% end %}
rescue ex
end
end
@@ -56,7 +133,17 @@ module Invidious::Routes::Images
name = env.params.url["name"]
url = env.request.resource
- headers = HTTP::Headers.new
+ headers = (
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ HTTP::Headers{":authority" => "i9.ytimg.com"}
+ else
+ HTTP::Headers.new
+ end
+ {% else %}
+ HTTP::Headers.new
+ {% end %}
+ )
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
@@ -64,10 +151,40 @@ module Invidious::Routes::Images
end
end
- begin
- get_ytimg_pool("i9").client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300 && response.status_code != 404
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
+ begin
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ YT_POOL.client &.get(url, headers) do |resp|
+ return request_proc.call(resp)
+ end
+ else
+ HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ {% end %}
rescue ex
end
end
@@ -106,15 +223,41 @@ module Invidious::Routes::Images
id = env.params.url["id"]
name = env.params.url["name"]
- headers = HTTP::Headers.new
+ headers = (
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ HTTP::Headers{":authority" => "i.ytimg.com"}
+ else
+ HTTP::Headers.new
+ end
+ {% else %}
+ HTTP::Headers.new
+ {% end %}
+ )
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
- if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
- name = thumb[:url] + ".jpg"
- break
- end
+ # Logic here is short enough that manually typing them out should be fine.
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
+ name = thumb[:url] + ".jpg"
+ break
+ end
+ else
+ if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+ name = thumb[:url] + ".jpg"
+ break
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+ name = thumb[:url] + ".jpg"
+ break
+ end
+ {% end %}
end
end
@@ -126,28 +269,41 @@ module Invidious::Routes::Images
end
end
- begin
- get_ytimg_pool("i").client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300 && response.status_code != 404
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
+ begin
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ YT_POOL.client &.get(url, headers) do |resp|
+ return request_proc.call(resp)
+ end
+ else
+ HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ {% end %}
rescue ex
end
end
-
- private def self.proxy_image(env, response)
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- return proxy_file(response, env)
- end
end
diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr
index e7de5018..99fc13a2 100644
--- a/src/invidious/routes/login.cr
+++ b/src/invidious/routes/login.cr
@@ -6,14 +6,14 @@ module Invidious::Routes::Login
user = env.get? "user"
- referer = get_referer(env, "/feed/subscriptions")
-
- return env.redirect referer if user
+ return env.redirect "/feed/subscriptions" if user
if !CONFIG.login_enabled
return error_template(400, "Login has been disabled by administrator.")
end
+ referer = get_referer(env, "/feed/subscriptions")
+
email = nil
password = nil
captcha = nil
@@ -21,6 +21,12 @@ module Invidious::Routes::Login
account_type = env.params.query["type"]?
account_type ||= "invidious"
+ captcha_type = env.params.query["captcha"]?
+ captcha_type ||= "image"
+
+ tfa = env.params.query["tfa"]?
+ prompt = nil
+
templated "user/login"
end
@@ -41,18 +47,283 @@ module Invidious::Routes::Login
account_type ||= "invidious"
case account_type
+ when "google"
+ tfa_code = env.params.body["tfa"]?.try &.lchop("G-")
+ traceback = IO::Memory.new
+
+ # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
+ begin
+ client = nil # Declare variable
+ {% unless flag?(:disable_quic) %}
+ client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL)
+ {% else %}
+ client = HTTP::Client.new(LOGIN_URL)
+ {% end %}
+
+ headers = HTTP::Headers.new
+
+ login_page = client.get("/ServiceLogin")
+ headers = login_page.cookies.add_request_headers(headers)
+
+ lookup_req = {
+ email, nil, [] of String, nil, "US", nil, nil, 2, false, true,
+ {nil, nil,
+ {2, 1, nil, 1,
+ "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
+ nil, [] of String, 4},
+ 1,
+ {nil, nil, [] of String},
+ nil, nil, nil, true,
+ },
+ email,
+ }.to_json
+
+ traceback << "Getting lookup..."
+
+ headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
+ headers["Google-Accounts-XSRF"] = "1"
+
+ response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
+ lookup_results = JSON.parse(response.body[5..-1])
+
+ traceback << "done, returned #{response.status_code}.
"
+
+ user_hash = lookup_results[0][2]
+
+ if token = env.params.body["token"]?
+ answer = env.params.body["answer"]?
+ captcha = {token, answer}
+ else
+ captcha = nil
+ end
+
+ challenge_req = {
+ user_hash, nil, 1, nil,
+ {1, nil, nil, nil,
+ {password, captcha, true},
+ },
+ {nil, nil,
+ {2, 1, nil, 1,
+ "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
+ nil, [] of String, 4},
+ 1,
+ {nil, nil, [] of String},
+ nil, nil, nil, true,
+ },
+ }.to_json
+
+ traceback << "Getting challenge..."
+
+ response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req))
+ headers = response.cookies.add_request_headers(headers)
+ challenge_results = JSON.parse(response.body[5..-1])
+
+ traceback << "done, returned #{response.status_code}.
"
+
+ headers["Cookie"] = URI.decode_www_form(headers["Cookie"])
+
+ if challenge_results[0][3]?.try &.== 7
+ return error_template(423, "Account has temporarily been disabled")
+ end
+
+ if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s
+ account_type = "google"
+ captcha_type = "image"
+ prompt = nil
+ tfa = tfa_code
+ captcha = {tokens: [token], question: ""}
+
+ return templated "user/login"
+ end
+
+ if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED"
+ return error_template(401, "Incorrect password")
+ end
+
+ prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]?
+ if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type
+ traceback << "Handling prompt #{prompt_type}.
"
+ case prompt_type
+ when "TWO_STEP_VERIFICATION"
+ prompt_type = 2
+ else # "LOGIN_CHALLENGE"
+ prompt_type = 4
+ end
+
+ # Prefer Authenticator app and SMS over unsupported protocols
+ if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2
+ tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0]
+
+ traceback << "Selecting challenge #{tfa[8]}..."
+ select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json
+
+ tl = challenge_results[1][2]
+
+ tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body
+ tfa = tfa[5..-1]
+ tfa = JSON.parse(tfa)[0][-1]
+
+ traceback << "done.
"
+ else
+ traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.
"
+ tfa = challenge_results[0][-1][0][0]
+ end
+
+ if tfa[5] == "QUOTA_EXCEEDED"
+ return error_template(423, "Quota exceeded, try again in a few hours")
+ end
+
+ if !tfa_code
+ account_type = "google"
+ captcha_type = "image"
+
+ case tfa[8]
+ when 6, 9
+ prompt = "Google verification code"
+ when 12
+ prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
+ when 15
+ prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
+ else
+ prompt = "Google verification code"
+ end
+
+ tfa = nil
+ captcha = nil
+ return templated "user/login"
+ end
+
+ tl = challenge_results[1][2]
+
+ request_type = tfa[8]
+ case request_type
+ when 6 # Authenticator app
+ tfa_req = {
+ user_hash, nil, 2, nil,
+ {6, nil, nil, nil, nil,
+ {tfa_code, false},
+ },
+ }.to_json
+ when 9 # Voice or text message
+ tfa_req = {
+ user_hash, nil, 2, nil,
+ {9, nil, nil, nil, nil, nil, nil, nil,
+ {nil, tfa_code, false, 2},
+ },
+ }.to_json
+ when 12 # Recovery email
+ tfa_req = {
+ user_hash, nil, 4, nil,
+ {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+ {tfa_code},
+ },
+ }.to_json
+ when 15 # Security question
+ tfa_req = {
+ user_hash, nil, 5, nil,
+ {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+ {tfa_code},
+ },
+ }.to_json
+ else
+ return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.")
+ end
+
+ traceback << "Submitting challenge..."
+
+ response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req))
+ headers = response.cookies.add_request_headers(headers)
+ challenge_results = JSON.parse(response.body[5..-1])
+
+ if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") ||
+ (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT")
+ return error_template(401, "Invalid TFA code")
+ end
+
+ traceback << "done.
"
+ end
+
+ traceback << "Logging in..."
+
+ location = URI.parse(challenge_results[0][-1][2].to_s)
+ cookies = HTTP::Cookies.from_client_headers(headers)
+
+ headers.delete("Content-Type")
+ headers.delete("Google-Accounts-XSRF")
+
+ loop do
+ if !location || location.path == "/ManageAccount"
+ break
+ end
+
+ # Occasionally there will be a second page after login confirming
+ # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle.
+
+ if location.path.starts_with? "/b/0/SmsAuthInterstitial"
+ traceback << "Unhandled dialog /b/0/SmsAuthInterstitial."
+ end
+
+ login = client.get(location.request_target, headers)
+
+ headers = login.cookies.add_request_headers(headers)
+ location = login.headers["Location"]?.try { |u| URI.parse(u) }
+ end
+
+ cookies = HTTP::Cookies.from_client_headers(headers)
+ sid = cookies["SID"]?.try &.value
+ if !sid
+ raise "Couldn't get SID."
+ end
+
+ user, sid = get_user(sid, headers)
+
+ # We are now logged in
+ traceback << "done.
"
+
+ host = URI.parse(env.request.headers["Host"]).host
+
+ cookies.each do |cookie|
+ cookie.secure = Invidious::User::Cookies::SECURE
+
+ if cookie.extension
+ cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host)
+ cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "")
+ end
+ env.response.cookies << cookie
+ end
+
+ if env.request.cookies["PREFS"]?
+ user.preferences = env.get("preferences").as(Preferences)
+ Invidious::Database::Users.update_preferences(user)
+
+ cookie = env.request.cookies["PREFS"]
+ cookie.expires = Time.utc(1990, 1, 1)
+ env.response.cookies << cookie
+ end
+
+ env.redirect referer
+ rescue ex
+ traceback.rewind
+ # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.")
+ error_message = %(#{ex.message}
Traceback:
#{traceback.gets_to_end})
+ return error_template(500, error_message)
+ end
when "invidious"
- if email.nil? || email.empty?
+ if !email
return error_template(401, "User ID is a required field")
end
- if password.nil? || password.empty?
+ if !password
return error_template(401, "Password is a required field")
end
user = Invidious::Database::Users.select(email: email)
if user
+ if !user.password
+ return error_template(400, "Please sign in using 'Log in with Google'")
+ end
+
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
Invidious::Database::SessionIDs.insert(sid, email)
@@ -85,14 +356,36 @@ module Invidious::Routes::Login
password = password.byte_slice(0, 55)
if CONFIG.captcha_enabled
+ captcha_type = env.params.body["captcha_type"]?
answer = env.params.body["answer"]?
+ change_type = env.params.body["change_type"]?
- account_type = "invidious"
- captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ if !captcha_type || change_type
+ if change_type
+ captcha_type = change_type
+ end
+ captcha_type ||= "image"
+
+ account_type = "invidious"
+ tfa = false
+ prompt = ""
+
+ if captcha_type == "image"
+ captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ else
+ captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
+ end
+
+ return templated "user/login"
+ end
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
- if answer
+ answer ||= ""
+ captcha_type ||= "image"
+
+ case captcha_type
+ when "image"
answer = answer.lstrip('0')
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
@@ -101,8 +394,27 @@ module Invidious::Routes::Login
rescue ex
return error_template(400, ex)
end
- else
- return templated "user/login"
+ else # "text"
+ answer = Digest::MD5.hexdigest(answer.downcase.strip)
+
+ if tokens.empty?
+ return error_template(500, "Erroneous CAPTCHA")
+ end
+
+ found_valid_captcha = false
+ error_exception = Exception.new
+ tokens.each do |tok|
+ begin
+ validate_request(tok, answer, env.request, HMAC_KEY, locale)
+ found_valid_captcha = true
+ rescue ex
+ error_exception = ex
+ end
+ end
+
+ if !found_valid_captcha
+ return error_template(500, error_exception)
+ end
end
end
@@ -169,4 +481,11 @@ module Invidious::Routes::Login
env.redirect referer
end
+
+ def self.captcha(env)
+ headers = HTTP::Headers{":authority" => "accounts.google.com"}
+ response = YT_POOL.client &.get(env.request.resource, headers)
+ env.response.headers["Content-Type"] = response.headers["Content-Type"]
+ response.body
+ end
end
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index 0b868755..d6bd9571 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -40,21 +40,7 @@ module Invidious::Routes::Misc
def self.cross_instance_redirect(env)
referer = get_referer(env)
-
- instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
- # Filter out the current instance
- other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
-
- if other_available_instances.empty?
- # If the current instance is the only one, use the redirect URL as fallback
- instance_url = "redirect.invidious.io"
- else
- # Select other random instance
- # Sample returns an array
- # Instances are packaged as {region, domain} in the instance list
- instance_url = other_available_instances.sample(1)[0][1]
- end
-
+ instance_url = fetch_random_instance
env.redirect "https://#{instance_url}#{referer}"
end
end
diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr
index 8922b740..272a3dc7 100644
--- a/src/invidious/routes/notifications.cr
+++ b/src/invidious/routes/notifications.cr
@@ -24,6 +24,50 @@ module Invidious::Routes::Notifications
user = user.as(User)
+ if !user.password
+ channel_req = {} of String => String
+
+ channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true"
+ channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || ""
+ channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true"
+
+ channel_req.reject! { |k, v| v != "true" && v != "false" }
+
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env.request.headers["Cookie"]
+
+ html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
+
+ cookies = HTTP::Cookies.from_client_headers(headers)
+ html.cookies.each do |cookie|
+ if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
+ if cookies[cookie.name]?
+ cookies[cookie.name] = cookie
+ else
+ cookies << cookie
+ end
+ end
+ end
+ headers = cookies.add_request_headers(headers)
+
+ if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
+ session_token = match["session_token"]
+ else
+ return env.redirect referer
+ end
+
+ headers["content-type"] = "application/x-www-form-urlencoded"
+ channel_req["session_token"] = session_token
+
+ subs = XML.parse_html(html.body)
+ subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel|
+ channel_id = channel.content.lstrip("/channel/").not_nil!
+ channel_req["channel_id"] = channel_id
+
+ YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
+ end
+ end
+
if redirect
env.redirect referer
else
diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr
index f2213da4..fe7e4e1c 100644
--- a/src/invidious/routes/playlists.cr
+++ b/src/invidious/routes/playlists.cr
@@ -163,20 +163,13 @@ module Invidious::Routes::Playlists
end
begin
- items = get_playlist_videos(playlist, offset: (page - 1) * 100)
+ videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
- items = [] of PlaylistVideo
+ videos = [] of PlaylistVideo
end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY)
- # Pagination
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/playlist?list=#{playlist.id}",
- current_page: page,
- show_next: (items.size == 100)
- )
-
templated "edit_playlist"
end
@@ -254,19 +247,11 @@ module Invidious::Routes::Playlists
begin
query = Invidious::Search::Query.new(env.params.query, :playlist, region)
- items = query.process.select(SearchVideo).map(&.as(SearchVideo))
+ videos = query.process.select(SearchVideo).map(&.as(SearchVideo))
rescue ex
- items = [] of SearchVideo
+ videos = [] of SearchVideo
end
- # Pagination
- query_encoded = URI.encode_www_form(query.try &.text || "", space_to_plus: true)
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/add_playlist_items?list=#{playlist.id}&q=#{query_encoded}",
- current_page: page,
- show_next: (items.size >= 20)
- )
-
env.set "add_playlist_items", plid
templated "add_playlist_items"
end
@@ -304,6 +289,23 @@ module Invidious::Routes::Playlists
end
end
+ if env.params.query["action_create_playlist"]?
+ action = "action_create_playlist"
+ elsif env.params.query["action_delete_playlist"]?
+ action = "action_delete_playlist"
+ elsif env.params.query["action_edit_playlist"]?
+ action = "action_edit_playlist"
+ elsif env.params.query["action_add_video"]?
+ action = "action_add_video"
+ video_id = env.params.query["video_id"]
+ elsif env.params.query["action_remove_video"]?
+ action = "action_remove_video"
+ elsif env.params.query["action_move_video_before"]?
+ action = "action_move_video_before"
+ else
+ return env.redirect referer
+ end
+
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
@@ -318,13 +320,21 @@ module Invidious::Routes::Playlists
end
end
- case action = env.params.query["action"]?
- when "add_video"
- if playlist.index.size >= CONFIG.playlist_length_limit
+ if !user.password
+ # TODO: Playlist stub, sync with YouTube for Google accounts
+ # playlist_ajax(playlist_id, action, env.request.headers)
+ end
+ email = user.email
+
+ case action
+ when "action_edit_playlist"
+ # TODO: Playlist stub
+ when "action_add_video"
+ if playlist.index.size >= 500
if redirect
- return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
+ return error_template(400, "Playlist cannot have more than 500 videos")
else
- return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
+ return error_json(400, "Playlist cannot have more than 500 videos")
end
end
@@ -356,14 +366,12 @@ module Invidious::Routes::Playlists
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
- when "remove_video"
+ when "action_remove_video"
index = env.params.query["set_video_id"]
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(playlist_id, index)
- when "move_video_before"
+ when "action_move_video_before"
# TODO: Playlist stub
- when nil
- return error_json(400, "Missing action")
else
return error_json(400, "Unsupported action #{action}")
end
@@ -402,13 +410,8 @@ module Invidious::Routes::Playlists
return error_template(500, ex)
end
- if playlist.is_a? InvidiousPlaylist
- page_count = (playlist.video_count / 100).to_i
- page_count += 1 if (playlist.video_count % 100) > 0
- else
- page_count = (playlist.video_count / 200).to_i
- page_count += 1 if (playlist.video_count % 200) > 0
- end
+ page_count = (playlist.video_count / 100).to_i
+ page_count += 1 if (playlist.video_count % 100) > 0
if page > page_count
return env.redirect "/playlist?list=#{plid}&page=#{page_count}"
@@ -419,11 +422,7 @@ module Invidious::Routes::Playlists
end
begin
- if playlist.is_a? InvidiousPlaylist
- items = get_playlist_videos(playlist, offset: (page - 1) * 100)
- else
- items = get_playlist_videos(playlist, offset: (page - 1) * 200)
- end
+ videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}")
end
@@ -432,13 +431,6 @@ module Invidious::Routes::Playlists
env.set "remove_playlist_items", plid
end
- # Pagination
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/playlist?list=#{playlist.id}",
- current_page: page,
- show_next: (page_count != 1 && page < page_count)
- )
-
templated "playlist"
end
diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr
index 39ca77c0..570cba69 100644
--- a/src/invidious/routes/preferences.cr
+++ b/src/invidious/routes/preferences.cr
@@ -27,10 +27,6 @@ module Invidious::Routes::PreferencesRoute
annotations_subscribed ||= "off"
annotations_subscribed = annotations_subscribed == "on"
- preload = env.params.body["preload"]?.try &.as(String)
- preload ||= "off"
- preload = preload == "on"
-
autoplay = env.params.body["autoplay"]?.try &.as(String)
autoplay ||= "off"
autoplay = autoplay == "on"
@@ -148,7 +144,6 @@ module Invidious::Routes::PreferencesRoute
preferences = Preferences.from_json({
annotations: annotations,
annotations_subscribed: annotations_subscribed,
- preload: preload,
autoplay: autoplay,
captions: captions,
comments: comments,
@@ -219,7 +214,7 @@ module Invidious::Routes::PreferencesRoute
statistics_enabled ||= "off"
CONFIG.statistics_enabled = statistics_enabled == "on"
- CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
+ CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
File.write("config/config.yml", CONFIG.to_yaml)
end
@@ -315,24 +310,6 @@ module Invidious::Routes::PreferencesRoute
response: error_template(415, "Invalid subscription file uploaded")
)
end
- when "import_youtube_pl"
- filename = part.filename || ""
- success = Invidious::User::Import.from_youtube_pl(user, body, filename, type)
-
- if !success
- haltf(env, status_code: 415,
- response: error_template(415, "Invalid playlist file uploaded")
- )
- end
- when "import_youtube_wh"
- filename = part.filename || ""
- success = Invidious::User::Import.from_youtube_wh(user, body, filename, type)
-
- if !success
- haltf(env, status_code: 415,
- response: error_template(415, "Invalid watch history file uploaded")
- )
- end
when "import_freetube"
Invidious::User::Import.from_freetube(user, body)
when "import_newpipe_subscriptions"
diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr
index b195c7b3..2a9705cf 100644
--- a/src/invidious/routes/search.cr
+++ b/src/invidious/routes/search.cr
@@ -51,39 +51,21 @@ module Invidious::Routes::Search
else
user = env.get? "user"
- # An URL was copy/pasted in the search box.
- # Redirect the user to the appropriate page.
- if query.url?
- return env.redirect UrlSanitizer.process(query.text).to_s
- end
-
begin
- if user
- items = query.process(user.as(User))
- else
- items = query.process
- end
+ videos = query.process
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex
return error_template(500, ex)
end
+ params = query.to_http_params
+ url_prev_page = "/search?#{params}&page=#{query.page - 1}"
+ url_next_page = "/search?#{params}&page=#{query.page + 1}"
+
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
- # Pagination
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/search?#{query.to_http_params}",
- current_page: query.page,
- show_next: (items.size >= 20)
- )
-
- if query.type == Invidious::Search::Query::Type::Channel
- env.set "search", "channel:#{query.channel} #{query.text}"
- else
- env.set "search", query.text
- end
-
+ env.set "search", query.text
templated "search"
end
end
@@ -105,18 +87,16 @@ module Invidious::Routes::Search
end
begin
- items = Invidious::Hashtag.fetch(hashtag, page)
+ videos = Invidious::Hashtag.fetch(hashtag, page)
rescue ex
return error_template(500, ex)
end
- # Pagination
+ params = env.params.query.empty? ? "" : "&#{env.params.query}"
+
hashtag_encoded = URI.encode_www_form(hashtag, space_to_plus: false)
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/hashtag/#{hashtag_encoded}",
- current_page: page,
- show_next: (items.size >= 60)
- )
+ url_prev_page = "/hashtag/#{hashtag_encoded}?page=#{page - 1}#{params}"
+ url_next_page = "/hashtag/#{hashtag_encoded}?page=#{page + 1}#{params}"
templated "hashtag"
end
diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr
index 1de655d2..7b1fa876 100644
--- a/src/invidious/routes/subscriptions.cr
+++ b/src/invidious/routes/subscriptions.cr
@@ -32,16 +32,29 @@ module Invidious::Routes::Subscriptions
end
end
+ if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1
+ action = "action_create_subscription_to_channel"
+ elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1
+ action = "action_remove_subscriptions"
+ else
+ return env.redirect referer
+ end
+
channel_id = env.params.query["c"]?
channel_id ||= ""
- case action = env.params.query["action"]?
- when "create_subscription_to_channel"
+ if !user.password
+ # Sync subscriptions with YouTube
+ subscribe_ajax(channel_id, action, env.request.headers)
+ end
+
+ case action
+ when "action_create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
- when "remove_subscriptions"
+ when "action_remove_subscriptions"
Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else
return error_json(400, "Unsupported action #{action}")
@@ -69,6 +82,14 @@ module Invidious::Routes::Subscriptions
user = user.as(User)
sid = sid.as(String)
+ if !user.password
+ # Refresh account
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env.request.headers["Cookie"]
+
+ user, sid = get_user(sid, headers)
+ end
+
action_takeout = env.params.query["action_takeout"]?.try &.to_i?
action_takeout ||= 0
action_takeout = action_takeout == 1
@@ -83,8 +104,33 @@ module Invidious::Routes::Subscriptions
if format == "json"
env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment"
+ playlists = Invidious::Database::Playlists.select_like_iv(user.email)
- return Invidious::User::Export.to_invidious(user)
+ return JSON.build do |json|
+ json.object do
+ json.field "subscriptions", user.subscriptions
+ json.field "watch_history", user.watched
+ json.field "preferences", user.preferences
+ json.field "playlists" do
+ json.array do
+ playlists.each do |playlist|
+ json.object do
+ json.field "title", playlist.title
+ json.field "description", html_to_content(playlist.description_html)
+ json.field "privacy", playlist.privacy.to_s
+ json.field "videos" do
+ json.array do
+ Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id|
+ json.string video_id
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
else
env.response.content_type = "application/xml"
env.response.headers["content-disposition"] = "attachment"
diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index 083087a9..560f9c19 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -21,7 +21,7 @@ module Invidious::Routes::VideoPlayback
end
# Sanity check, to avoid being used as an open proxy
- if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/)
+ if !host.matches?(/[\w-]+.googlevideo.com/)
return error_template(400, "Invalid \"host\" parameter.")
end
@@ -35,15 +35,7 @@ module Invidious::Routes::VideoPlayback
end
end
- # See: https://github.com/iv-org/invidious/issues/3302
- range_header = env.request.headers["Range"]?
- sq = query_params["sq"]?
- if range_header.nil? && sq.nil?
- range_for_head = query_params["range"]? || "0-640"
- headers["Range"] = "bytes=#{range_for_head}"
- end
-
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
@@ -58,7 +50,7 @@ module Invidious::Routes::VideoPlayback
if new_host != host
host = new_host
client.close
- client = make_client(URI.parse(new_host), region, force_resolve: true)
+ client = make_client(URI.parse(new_host), region)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
@@ -72,23 +64,15 @@ module Invidious::Routes::VideoPlayback
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region)
rescue ex
error = ex.message
end
end
- # Remove the Range header added previously.
- headers.delete("Range") if range_header.nil?
-
- playback_statistics = get_playback_statistic()
- playback_statistics["totalRequests"] += 1
-
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
- else
- playback_statistics["successfulRequests"] += 1
end
if url.includes? "&file=seg.ts"
@@ -107,8 +91,14 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
- return env.redirect url
+ location = URI.parse(location)
+ location = "#{location.request_target}&host=#{location.host}"
+
+ if region
+ location += "®ion=#{region}"
+ end
+
+ return env.redirect location
end
IO.copy(resp.body_io, env.response)
@@ -132,7 +122,7 @@ module Invidious::Routes::VideoPlayback
end
# TODO: Record bytes written so we can restart after a chunk fails
- loop do
+ while true
if !range_end && content_length
range_end = content_length
end
@@ -165,13 +155,10 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
+ location = URI.parse(location)
+ location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
- if title = query_params["title"]?
- url = "#{url}&title=#{URI.encode_www_form(title)}"
- end
-
- env.redirect url
+ env.redirect location
break
end
@@ -200,7 +187,7 @@ module Invidious::Routes::VideoPlayback
break
else
client.close
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region)
end
end
@@ -257,11 +244,6 @@ module Invidious::Routes::VideoPlayback
# YouTube /videoplayback links expire after 6 hours,
# so we have a mechanism here to redirect to the latest version
def self.latest_version(env)
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
- end
-
id = env.params.query["id"]?
itag = env.params.query["itag"]?.try &.to_i?
@@ -270,7 +252,7 @@ module Invidious::Routes::VideoPlayback
return error_template(400, "Invalid video ID")
end
- if !itag.nil? && (itag <= 0 || itag >= 1000)
+ if itag.nil? || itag <= 0 || itag >= 1000
return error_template(400, "Invalid itag")
end
@@ -291,11 +273,7 @@ module Invidious::Routes::VideoPlayback
return error_template(500, ex)
end
- if itag.nil?
- fmt = video.fmt_stream[-1]?
- else
- fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
- end
+ fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
url = fmt.try &.["url"]?.try &.as_s
if !url
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
index e777b3f1..fe1d8e54 100644
--- a/src/invidious/routes/watch.cr
+++ b/src/invidious/routes/watch.cr
@@ -30,6 +30,14 @@ module Invidious::Routes::Watch
return env.redirect "/"
end
+ embed_link = "/embed/#{id}"
+ if env.params.query.size > 1
+ embed_params = HTTP::Params.parse(env.params.query.to_s)
+ embed_params.delete_all("v")
+ embed_link += "?"
+ embed_link += embed_params.to_s
+ end
+
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(env.params.query, plid, id)
@@ -53,6 +61,8 @@ module Invidious::Routes::Watch
begin
video = get_video(id, region: params.region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
LOGGER.error("get_video not found: #{id} : #{ex.message}")
return error_template(404, ex)
@@ -68,11 +78,11 @@ module Invidious::Routes::Watch
end
env.params.query.delete_all("iv_load_policy")
- if watched && preferences.watch_history
+ if watched && preferences.watch_history && !watched.includes? id
Invidious::Database::Users.mark_watched(user.as(User), id)
end
- if CONFIG.enable_user_notifications && notifications && notifications.includes? id
+ if notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
@@ -87,31 +97,31 @@ module Invidious::Routes::Watch
if source == "youtube"
begin
- comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
rescue ex
if preferences.comments[1] == "reddit"
- comments, reddit_thread = Comments.fetch_reddit(id)
- comment_html = Frontend::Comments.template_reddit(comments, locale)
+ comments, reddit_thread = fetch_reddit_comments(id)
+ comment_html = template_reddit_comments(comments, locale)
- comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
- comment_html = Comments.replace_links(comment_html)
+ comment_html = fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = replace_links(comment_html)
end
end
elsif source == "reddit"
begin
- comments, reddit_thread = Comments.fetch_reddit(id)
- comment_html = Frontend::Comments.template_reddit(comments, locale)
+ comments, reddit_thread = fetch_reddit_comments(id)
+ comment_html = template_reddit_comments(comments, locale)
- comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
- comment_html = Comments.replace_links(comment_html)
+ comment_html = fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = replace_links(comment_html)
rescue ex
if preferences.comments[1] == "youtube"
- comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
end
end
else
- comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
comment_html ||= ""
@@ -121,12 +131,10 @@ module Invidious::Routes::Watch
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
end
- # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -192,14 +200,6 @@ module Invidious::Routes::Watch
captions: video.captions
)
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- env.response.headers["Content-Security-Policy"] =
- env.response.headers["Content-Security-Policy"]
- .gsub("media-src", "media-src #{invidious_companion.public_url}")
- .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
- end
-
templated "watch"
end
@@ -251,10 +251,20 @@ module Invidious::Routes::Watch
end
end
- case action = env.params.query["action"]?
- when "mark_watched"
- Invidious::Database::Users.mark_watched(user, id)
- when "mark_unwatched"
+ if env.params.query["action_mark_watched"]?
+ action = "action_mark_watched"
+ elsif env.params.query["action_mark_unwatched"]?
+ action = "action_mark_unwatched"
+ else
+ return env.redirect referer
+ end
+
+ case action
+ when "action_mark_watched"
+ if !user.watched.includes? id
+ Invidious::Database::Users.mark_watched(user, id)
+ end
+ when "action_mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
else
return error_json(400, "Unsupported action #{action}")
@@ -277,12 +287,6 @@ module Invidious::Routes::Watch
return error_template(400, "Invalid clip ID") if response["error"]?
if video_id = response.dig?("endpoint", "watchEndpoint", "videoId")
- if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
- start_time, end_time, _ = parse_clip_parameters(params)
- env.params.query["start"] = start_time.to_s if start_time != nil
- env.params.query["end"] = end_time.to_s if end_time != nil
- end
-
return env.redirect "/watch?v=#{video_id}&#{env.params.query}"
else
return error_template(404, "The requested clip doesn't exist")
@@ -293,9 +297,6 @@ module Invidious::Routes::Watch
if CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.")
end
- if CONFIG.invidious_companion.present?
- return error_template(403, "Downloads should be routed through Companion when present")
- end
title = env.params.body["title"]? || ""
video_id = env.params.body["id"]? || ""
@@ -325,9 +326,10 @@ module Invidious::Routes::Watch
env.params.query["label"] = URI.decode_www_form(label.as_s)
return Invidious::Routes::API::V1::Videos.captions(env)
- elsif itag = download_widget["itag"]?.try &.as_i.to_s
+ elsif itag = download_widget["itag"]?.try &.as_i
# URL params specific to /latest_version
env.params.query["id"] = video_id
+ env.params.query["itag"] = itag.to_s
env.params.query["title"] = filename
env.params.query["local"] = "true"
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 46b71f1f..f409f13c 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -37,9 +37,7 @@ module Invidious::Routing
get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get
post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post
- if CONFIG.enable_user_notifications
- get "/modify_notifications", Routes::Notifications, :modify
- end
+ get "/modify_notifications", Routes::Notifications, :modify
{% end %}
self.register_image_routes
@@ -57,6 +55,7 @@ module Invidious::Routing
get "/login", Routes::Login, :login_page
post "/login", Routes::Login, :login
post "/signout", Routes::Login, :signout
+ get "/Captcha", Routes::Login, :captcha
# User preferences
get "/preferences", Routes::PreferencesRoute, :show
@@ -116,52 +115,29 @@ module Invidious::Routing
get "/channel/:ucid", Routes::Channels, :home
get "/channel/:ucid/home", Routes::Channels, :home
get "/channel/:ucid/videos", Routes::Channels, :videos
- get "/channel/:ucid/shorts", Routes::Channels, :shorts
- get "/channel/:ucid/streams", Routes::Channels, :streams
- get "/channel/:ucid/podcasts", Routes::Channels, :podcasts
- get "/channel/:ucid/releases", Routes::Channels, :releases
- get "/channel/:ucid/courses", Routes::Channels, :courses
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
- get "/channel/:ucid/posts", Routes::Channels, :community
- get "/channel/:ucid/channels", Routes::Channels, :channels
get "/channel/:ucid/about", Routes::Channels, :about
-
get "/channel/:ucid/live", Routes::Channels, :live
get "/user/:user/live", Routes::Channels, :live
get "/c/:user/live", Routes::Channels, :live
- get "/post/:id", Routes::Channels, :post
- # Channel catch-all, to redirect future routes to the channel's home
- # NOTE: defined last in order to be processed after the other routes
- get "/channel/:ucid/*", Routes::Channels, :redirect_home
-
- # /c/LinusTechTips
- get "/c/:user", Routes::Channels, :brand_redirect
- get "/c/:user/:tab", Routes::Channels, :brand_redirect
-
- # /user/linustechtips (Not always the same as /c/)
- get "/user/:user", Routes::Channels, :brand_redirect
- get "/user/:user/:tab", Routes::Channels, :brand_redirect
-
- # /@LinusTechTips (Handle)
- get "/@:user", Routes::Channels, :brand_redirect
- get "/@:user/:tab", Routes::Channels, :brand_redirect
-
- # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow
- get "/attribution_link", Routes::Channels, :brand_redirect
- get "/attribution_link/:tab", Routes::Channels, :brand_redirect
-
- # /profile?user=linustechtips
- get "/profile", Routes::Channels, :profile
- get "/profile/*", Routes::Channels, :profile
+ ["", "/videos", "/playlists", "/community", "/about"].each do |path|
+ # /c/LinusTechTips
+ get "/c/:user#{path}", Routes::Channels, :brand_redirect
+ # /user/linustechtips | Not always the same as /c/
+ get "/user/:user#{path}", Routes::Channels, :brand_redirect
+ # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow
+ get "/attribution_link#{path}", Routes::Channels, :brand_redirect
+ # /profile?user=linustechtips
+ get "/profile/#{path}", Routes::Channels, :profile
+ end
end
def register_watch_routes
get "/watch", Routes::Watch, :handle
post "/watch_ajax", Routes::Watch, :mark_watched
get "/watch/:id", Routes::Watch, :redirect
- get "/live/:id", Routes::Watch, :redirect
get "/shorts/:id", Routes::Watch, :redirect
get "/clip/:clip", Routes::Watch, :clip
get "/w/:id", Routes::Watch, :redirect
@@ -237,8 +213,6 @@ module Invidious::Routing
get "/api/v1/captions/:id", {{namespace}}::Videos, :captions
get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
- get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
- get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts
# Feeds
get "/api/v1/trending", {{namespace}}::Feeds, :trending
@@ -246,22 +220,10 @@ module Invidious::Routing
# Channels
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
- get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest
- get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos
- get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
- get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
- get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
- get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
- get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses
- get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
- get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
- get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community
- get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
- get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search
-
- # Posts
- get "/api/v1/post/:id", {{namespace}}::Channels, :post
- get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments
+ {% for route in {"videos", "latest", "playlists", "community", "search"} %}
+ get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
+ get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
+ {% end %}
# 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community
get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect
@@ -270,22 +232,17 @@ module Invidious::Routing
# Search
get "/api/v1/search", {{namespace}}::Search, :search
get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions
- get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag
-
# Authenticated
+ # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr
+ #
+ # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+
get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
- get "/api/v1/auth/export/invidious", {{namespace}}::Authenticated, :export_invidious
- post "/api/v1/auth/import/invidious", {{namespace}}::Authenticated, :import_invidious
-
- get "/api/v1/auth/history", {{namespace}}::Authenticated, :get_history
- post "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_watched
- delete "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_unwatched
- delete "/api/v1/auth/history", {{namespace}}::Authenticated, :clear_history
-
get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed
get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions
@@ -303,17 +260,14 @@ module Invidious::Routing
post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token
post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token
- if CONFIG.enable_user_notifications
- get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
- post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
- end
+ get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
# Misc
get "/api/v1/stats", {{namespace}}::Misc, :stats
get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist
get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist
get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes
- get "/api/v1/resolveurl", {{namespace}}::Misc, :resolve_url
{% end %}
end
end
diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr
index bc2715cf..c2b5c758 100644
--- a/src/invidious/search/filters.cr
+++ b/src/invidious/search/filters.cr
@@ -75,7 +75,7 @@ module Invidious::Search
@type : Type = Type::All,
@duration : Duration = Duration::None,
@features : Features = Features::None,
- @sort : Sort = Sort::Relevance,
+ @sort : Sort = Sort::Relevance
)
end
@@ -300,9 +300,9 @@ module Invidious::Search
object["9:varint"] = ((page - 1) * 20).to_i64
end
- # Prevent censoring of self harm topics
- # See https://github.com/iv-org/invidious/issues/4398
- object["30:varint"] = 1.to_i64
+ # If the object is empty, return an empty string,
+ # otherwise encode to protobuf then to base64
+ return "" if object.empty?
return object
.try { |i| Protodec::Any.cast_json(i) }
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index 25edb936..d1409c06 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -9,8 +9,7 @@ module Invidious::Search
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
- items, _ = extract_items(initial_data)
- return items.reject!(Category)
+ return extract_items(initial_data)
end
# Search a youtube channel
@@ -31,8 +30,17 @@ module Invidious::Search
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
- items, _ = extract_items(response_json, "", ucid)
- return items.reject!(Category)
+ continuation_items = response_json["onResponseReceivedActions"]?
+ .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
+
+ return [] of SearchItem if !continuation_items
+
+ items = [] of SearchItem
+ continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
+ extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
+ end
+
+ return items
end
# Search inside of user subscriptions
diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr
index 94a92e23..24e79609 100644
--- a/src/invidious/search/query.cr
+++ b/src/invidious/search/query.cr
@@ -20,9 +20,6 @@ module Invidious::Search
property region : String?
property channel : String = ""
- # Flag that indicates if the smart search features have been disabled.
- @inhibit_ssf : Bool = false
-
# Return true if @raw_query is either `nil` or empty
private def empty_raw_query?
return @raw_query.empty?
@@ -47,22 +44,14 @@ module Invidious::Search
def initialize(
params : HTTP::Params,
@type : Type = Type::Regular,
- @region : String? = nil,
+ @region : String? = nil
)
# Get the raw search query string (common to all search types). In
# Regular search mode, also look for the `search_query` URL parameter
- _raw_query = params["q"]?
- _raw_query ||= params["search_query"]? if @type.regular?
- _raw_query ||= ""
-
- # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs.
- @raw_query = _raw_query.strip
-
- # Check for smart features (ex: URL search) inhibitor (backslash).
- # If inhibitor is present, remove it.
- if @raw_query.starts_with?('\\')
- @inhibit_ssf = true
- @raw_query = @raw_query[1..]
+ if @type.regular?
+ @raw_query = params["q"]? || params["search_query"]? || ""
+ else
+ @raw_query = params["q"]? || ""
end
# Get the page number (also common to all search types)
@@ -96,7 +85,7 @@ module Invidious::Search
@filters = Filters.from_iv_params(params)
@channel = params["channel"]? || ""
- if @filters.default? && @raw_query.index(/\w:\w/)
+ if @filters.default? && @raw_query.includes?(':')
# Parse legacy filters from query
@filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query)
else
@@ -124,7 +113,7 @@ module Invidious::Search
case @type
when .regular?, .playlist?
- items = Processors.regular(self)
+ items = unnest_items(Processors.regular(self))
#
when .channel?
items = Processors.channel(self)
@@ -148,21 +137,25 @@ module Invidious::Search
return params
end
- # Checks if the query is a standalone URL
- def url? : Bool
- # If the smart features have been inhibited, don't go further.
- return false if @inhibit_ssf
+ # TODO: clean code
+ private def unnest_items(all_items) : Array(SearchItem)
+ items = [] of SearchItem
- # Only supported in regular search mode
- return false if !@type.regular?
+ # Light processing to flatten search results out of Categories.
+ # They should ideally be supported in the future.
+ all_items.each do |i|
+ if i.is_a? Category
+ i.contents.each do |nest_i|
+ if !nest_i.is_a? Video
+ items << nest_i
+ end
+ end
+ else
+ items << i
+ end
+ end
- # If filters are present, that's a regular search
- return false if !@filters.default?
-
- # Simple heuristics: domain name
- return @raw_query.starts_with?(
- /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\//
- )
+ return items
end
end
end
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index d14cde5d..1f957081 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -4,12 +4,11 @@ def fetch_trending(trending_type, region, locale)
plid = nil
- case trending_type.try &.downcase
- when "music"
+ if trending_type == "Music"
params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D"
- when "gaming"
+ elsif trending_type == "Gaming"
params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D"
- when "movies"
+ elsif trending_type == "Movies"
params = "4gIKGgh0cmFpbGVycw%3D%3D"
else # Default
params = ""
@@ -17,26 +16,7 @@ def fetch_trending(trending_type, region, locale)
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.browse("FEtrending", params: params, client_config: client_config)
+ trending = extract_videos(initial_data)
- items, _ = extract_items(initial_data)
-
- extracted = [] of SearchItem
-
- deduplicate = items.size > 1
-
- items.each do |itm|
- if itm.is_a?(Category)
- # Ignore the smaller categories, as they generally contain a sponsored
- # channel, which brings a lot of noise on the trending page.
- # See: https://github.com/iv-org/invidious/issues/2989
- next if (itm.contents.size < 24 && deduplicate)
-
- extracted.concat itm.contents.select(SearchItem)
- else
- extracted << itm
- end
- end
-
- # Deduplicate items before returning results
- return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
+ return {trending, plid}
end
diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr
index b175c3b9..8a0f67e5 100644
--- a/src/invidious/user/captcha.cr
+++ b/src/invidious/user/captcha.cr
@@ -4,6 +4,8 @@ struct Invidious::User
module Captcha
extend self
+ private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com")
+
def generate_image(key)
second = Random::Secure.rand(12)
second_angle = second * 30
@@ -58,5 +60,19 @@ struct Invidious::User
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)},
}
end
+
+ def generate_text(key)
+ response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body)
+ response = JSON.parse(response)
+
+ tokens = response["a"].as_a.map do |answer|
+ generate_response(answer.as_s, {":login"}, key, use_nonce: true)
+ end
+
+ return {
+ question: response["q"].as_s,
+ tokens: tokens,
+ }
+ end
end
end
diff --git a/src/invidious/user/exports.cr b/src/invidious/user/exports.cr
deleted file mode 100644
index b52503c9..00000000
--- a/src/invidious/user/exports.cr
+++ /dev/null
@@ -1,35 +0,0 @@
-struct Invidious::User
- module Export
- extend self
-
- def to_invidious(user : User)
- playlists = Invidious::Database::Playlists.select_like_iv(user.email)
-
- return JSON.build do |json|
- json.object do
- json.field "subscriptions", user.subscriptions
- json.field "watch_history", user.watched
- json.field "preferences", user.preferences
- json.field "playlists" do
- json.array do
- playlists.each do |playlist|
- json.object do
- json.field "title", playlist.title
- json.field "description", html_to_content(playlist.description_html)
- json.field "privacy", playlist.privacy.to_s
- json.field "videos" do
- json.array do
- Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: CONFIG.playlist_length_limit).each do |video_id|
- json.string video_id
- end
- end
- end
- end
- end
- end
- end
- end
- end
- end
- end # module
-end
diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr
index 007eb666..f8b9e4e4 100644
--- a/src/invidious/user/imports.cr
+++ b/src/invidious/user/imports.cr
@@ -6,7 +6,7 @@ struct Invidious::User
# Parse a youtube CSV subscription file
def parse_subscription_export_csv(csv_content : String)
- rows = CSV.new(csv_content.strip('\n'), headers: true)
+ rows = CSV.new(csv_content, headers: true)
subscriptions = Array(String).new
# Counter to limit the amount of imports.
@@ -30,60 +30,6 @@ struct Invidious::User
return subscriptions
end
- def parse_playlist_export_csv(user : User, raw_input : String)
- # Split the input into head and body content
- raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true)
-
- # Create the playlist from the head content
- csv_head = CSV.new(raw_head.strip('\n'), headers: true)
- csv_head.next
- title = csv_head[4]
- description = csv_head[5]
- visibility = csv_head[6]
-
- if visibility.compare("Public", case_insensitive: true) == 0
- privacy = PlaylistPrivacy::Public
- else
- privacy = PlaylistPrivacy::Private
- end
-
- playlist = create_playlist(title, privacy, user)
- Invidious::Database::Playlists.update_description(playlist.id, description)
-
- # Add each video to the playlist from the body content
- csv_body = CSV.new(raw_body.strip('\n'), headers: true)
- csv_body.each do |row|
- video_id = row[0]
- if playlist
- next if !video_id
- next if video_id == "Video Id"
-
- begin
- video = get_video(video_id)
- rescue ex
- next
- end
-
- playlist_video = PlaylistVideo.new({
- title: video.title,
- id: video.id,
- author: video.author,
- ucid: video.ucid,
- length_seconds: video.length_seconds,
- published: video.published,
- plid: playlist.id,
- live_now: video.live_now,
- index: Random::Secure.rand(0_i64..Int64::MAX),
- })
-
- Invidious::Database::PlaylistVideos.insert(playlist_video)
- Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
- end
- end
-
- return playlist
- end
-
# -------------------
# Invidious
# -------------------
@@ -102,7 +48,7 @@ struct Invidious::User
if data["watch_history"]?
user.watched += data["watch_history"].as_a.map(&.as_s)
- user.watched.reverse!.uniq!.reverse!
+ user.watched.uniq!
Invidious::Database::Users.update_watch_history(user)
end
@@ -115,7 +61,7 @@ struct Invidious::User
playlists.each do |item|
title = item["title"]?.try &.as_s?.try &.delete("<>")
description = item["description"]?.try &.as_s?.try &.delete("\r")
- privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state }
+ privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
next if !title
next if !description
@@ -124,16 +70,14 @@ struct Invidious::User
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
- item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
- if idx > CONFIG.playlist_length_limit
- raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
- end
+ videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
+ raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
video_id = video_id.try &.as_s?
next if !video_id
begin
- video = get_video(video_id, false)
+ video = get_video(video_id)
rescue ex
next
end
@@ -161,7 +105,7 @@ struct Invidious::User
# Youtube
# -------------------
- private def opml?(mimetype : String, extension : String)
+ private def is_opml?(mimetype : String, extension : String)
opml_mimetypes = [
"application/xml",
"text/xml",
@@ -179,10 +123,10 @@ struct Invidious::User
def from_youtube(user : User, body : String, filename : String, type : String) : Bool
extension = filename.split(".").last
- if opml?(type, extension)
+ if is_opml?(type, extension)
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
- channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0]
+ channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body)
@@ -203,41 +147,6 @@ struct Invidious::User
return true
end
- def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool
- extension = filename.split(".").last
-
- if extension == "csv" || type == "text/csv"
- playlist = parse_playlist_export_csv(user, body)
- if playlist
- return true
- else
- return false
- end
- else
- return false
- end
- end
-
- def from_youtube_wh(user : User, body : String, filename : String, type : String) : Bool
- extension = filename.split(".").last
-
- if extension == "json" || type == "application/json"
- data = JSON.parse(body)
- watched = data.as_a.compact_map do |item|
- next unless url = item["titleUrl"]?
- next unless match = url.as_s.match(/\?v=(?[a-zA-Z0-9_-]+)$/)
- match["video_id"]
- end
- watched.reverse! # YouTube have newest first
- user.watched += watched
- user.watched.uniq!
- Invidious::Database::Users.update_watch_history(user)
- return true
- else
- return false
- end
- end
-
# -------------------
# Freetube
# -------------------
@@ -248,12 +157,8 @@ struct Invidious::User
subs = matches.map(&.["channel_id"])
if subs.empty?
- profiles = body.split('\n', remove_empty: true)
- profiles.each do |profile|
- if data = JSON.parse(profile)["subscriptions"]?
- subs += data.as_a.map(&.["id"].as_s)
- end
- end
+ data = JSON.parse(body)["subscriptions"]
+ subs = data.as_a.map(&.["id"].as_s)
end
user.subscriptions += subs
@@ -290,39 +195,42 @@ struct Invidious::User
end
def from_newpipe(user : User, body : String) : Bool
- Compress::Zip::File.open(IO::Memory.new(body), true) do |file|
- entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" }
- return false if entry.nil?
- entry.open do |file_io|
- # Ensure max size of 4MB
- io_sized = IO::Sized.new(file_io, 0x400000)
+ io = IO::Memory.new(body)
- begin
- temp = File.tempfile(".db") do |tempfile|
- begin
- File.write(tempfile.path, io_sized.gets_to_end)
- rescue
- return false
- end
+ Compress::Zip::File.open(io) do |file|
+ file.entries.each do |entry|
+ entry.open do |file_io|
+ # Ensure max size of 4MB
+ io_sized = IO::Sized.new(file_io, 0x400000)
- DB.open("sqlite3://" + tempfile.path) do |db|
- user.watched += db.query_all("SELECT url FROM streams", as: String)
- .map(&.lchop("https://www.youtube.com/watch?v="))
+ next if entry.filename != "newpipe.db"
- user.watched.uniq!
- Invidious::Database::Users.update_watch_history(user)
+ tempfile = File.tempfile(".db")
- user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
- .map(&.lchop("https://www.youtube.com/channel/"))
-
- user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions)
-
- Invidious::Database::Users.update_subscriptions(user)
- end
+ begin
+ File.write(tempfile.path, io_sized.gets_to_end)
+ rescue
+ return false
end
- ensure
- temp.delete if !temp.nil?
+
+ db = DB.open("sqlite3://" + tempfile.path)
+
+ user.watched += db.query_all("SELECT url FROM streams", as: String)
+ .map(&.lchop("https://www.youtube.com/watch?v="))
+
+ user.watched.uniq!
+ Invidious::Database::Users.update_watch_history(user)
+
+ user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
+ .map(&.lchop("https://www.youtube.com/channel/"))
+
+ user.subscriptions.uniq!
+ user.subscriptions = get_batch_channels(user.subscriptions)
+
+ Invidious::Database::Users.update_subscriptions(user)
+
+ db.close
+ tempfile.delete
end
end
end
diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr
index 0a8525f3..b3059403 100644
--- a/src/invidious/user/preferences.cr
+++ b/src/invidious/user/preferences.cr
@@ -4,7 +4,6 @@ struct Preferences
property annotations : Bool = CONFIG.default_user_preferences.annotations
property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
- property preload : Bool = CONFIG.default_user_preferences.preload
property autoplay : Bool = CONFIG.default_user_preferences.autoplay
property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index 65566d20..b763596b 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -3,6 +3,75 @@ require "crypto/bcrypt/password"
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
+def get_user(sid, headers, refresh = true)
+ if email = Invidious::Database::SessionIDs.select_email(sid)
+ user = Invidious::Database::Users.select!(email: email)
+
+ if refresh && Time.utc - user.updated > 1.minute
+ user, sid = fetch_user(sid, headers)
+
+ Invidious::Database::Users.insert(user, update_on_conflict: true)
+ Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
+
+ begin
+ view_name = "subscriptions_#{sha256(user.email)}"
+ PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
+ rescue ex
+ end
+ end
+ else
+ user, sid = fetch_user(sid, headers)
+
+ Invidious::Database::Users.insert(user, update_on_conflict: true)
+ Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
+
+ begin
+ view_name = "subscriptions_#{sha256(user.email)}"
+ PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
+ rescue ex
+ end
+ end
+
+ return user, sid
+end
+
+def fetch_user(sid, headers)
+ feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
+ feed = XML.parse_html(feed.body)
+
+ channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
+ if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
+ nil
+ else
+ channel["href"].lstrip("/channel/")
+ end
+ end
+
+ channels = get_batch_channels(channels)
+
+ email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
+ if email
+ email = email.content.strip
+ else
+ email = ""
+ end
+
+ token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
+
+ user = Invidious::User.new({
+ updated: Time.utc,
+ notifications: [] of String,
+ subscriptions: channels,
+ email: email,
+ preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
+ password: nil,
+ token: token,
+ watched: [] of String,
+ feed_needs_update: true,
+ })
+ return user, sid
+end
+
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
@@ -22,6 +91,38 @@ def create_user(sid, email, password)
return user, sid
end
+def subscribe_ajax(channel_id, action, env_headers)
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env_headers["Cookie"]
+
+ html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
+
+ cookies = HTTP::Cookies.from_client_headers(headers)
+ html.cookies.each do |cookie|
+ if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
+ if cookies[cookie.name]?
+ cookies[cookie.name] = cookie
+ else
+ cookies << cookie
+ end
+ end
+ end
+ headers = cookies.add_request_headers(headers)
+
+ if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
+ session_token = match["session_token"]
+
+ headers["content-type"] = "application/x-www-form-urlencoded"
+
+ post_req = {
+ session_token: session_token,
+ }
+ post_url = "/subscription_ajax?#{action}=1&c=#{channel_id}"
+
+ YT_POOL.client &.post(post_url, headers, form: post_req)
+ end
+end
+
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 348a0a66..c0ed6e85 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -1,22 +1,280 @@
-enum VideoType
- Video
- Livestream
- Scheduled
+CAPTION_LANGUAGES = {
+ "",
+ "English",
+ "English (auto-generated)",
+ "English (United Kingdom)",
+ "English (United States)",
+ "Afrikaans",
+ "Albanian",
+ "Amharic",
+ "Arabic",
+ "Armenian",
+ "Azerbaijani",
+ "Bangla",
+ "Basque",
+ "Belarusian",
+ "Bosnian",
+ "Bulgarian",
+ "Burmese",
+ "Cantonese (Hong Kong)",
+ "Catalan",
+ "Cebuano",
+ "Chinese",
+ "Chinese (China)",
+ "Chinese (Hong Kong)",
+ "Chinese (Simplified)",
+ "Chinese (Taiwan)",
+ "Chinese (Traditional)",
+ "Corsican",
+ "Croatian",
+ "Czech",
+ "Danish",
+ "Dutch",
+ "Dutch (auto-generated)",
+ "Esperanto",
+ "Estonian",
+ "Filipino",
+ "Finnish",
+ "French",
+ "French (auto-generated)",
+ "Galician",
+ "Georgian",
+ "German",
+ "German (auto-generated)",
+ "Greek",
+ "Gujarati",
+ "Haitian Creole",
+ "Hausa",
+ "Hawaiian",
+ "Hebrew",
+ "Hindi",
+ "Hmong",
+ "Hungarian",
+ "Icelandic",
+ "Igbo",
+ "Indonesian",
+ "Indonesian (auto-generated)",
+ "Interlingue",
+ "Irish",
+ "Italian",
+ "Italian (auto-generated)",
+ "Japanese",
+ "Japanese (auto-generated)",
+ "Javanese",
+ "Kannada",
+ "Kazakh",
+ "Khmer",
+ "Korean",
+ "Korean (auto-generated)",
+ "Kurdish",
+ "Kyrgyz",
+ "Lao",
+ "Latin",
+ "Latvian",
+ "Lithuanian",
+ "Luxembourgish",
+ "Macedonian",
+ "Malagasy",
+ "Malay",
+ "Malayalam",
+ "Maltese",
+ "Maori",
+ "Marathi",
+ "Mongolian",
+ "Nepali",
+ "Norwegian Bokmål",
+ "Nyanja",
+ "Pashto",
+ "Persian",
+ "Polish",
+ "Portuguese",
+ "Portuguese (auto-generated)",
+ "Portuguese (Brazil)",
+ "Punjabi",
+ "Romanian",
+ "Russian",
+ "Russian (auto-generated)",
+ "Samoan",
+ "Scottish Gaelic",
+ "Serbian",
+ "Shona",
+ "Sindhi",
+ "Sinhala",
+ "Slovak",
+ "Slovenian",
+ "Somali",
+ "Southern Sotho",
+ "Spanish",
+ "Spanish (auto-generated)",
+ "Spanish (Latin America)",
+ "Spanish (Mexico)",
+ "Spanish (Spain)",
+ "Sundanese",
+ "Swahili",
+ "Swedish",
+ "Tajik",
+ "Tamil",
+ "Telugu",
+ "Thai",
+ "Turkish",
+ "Turkish (auto-generated)",
+ "Ukrainian",
+ "Urdu",
+ "Uzbek",
+ "Vietnamese",
+ "Vietnamese (auto-generated)",
+ "Welsh",
+ "Western Frisian",
+ "Xhosa",
+ "Yiddish",
+ "Yoruba",
+ "Zulu",
+}
+
+REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"}
+
+# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
+VIDEO_FORMATS = {
+ "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
+ "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
+ "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
+ "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
+ "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
+ "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+
+ "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
+ "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+
+ # 3D videos
+ "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+
+ # Apple HTTP Live Streaming
+ "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
+ "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
+ "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
+
+ # DASH mp4 video
+ "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
+ "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
+ "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
+ "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
+ "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
+ "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
+ "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
+ "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
+ "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
+ "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
+ "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
+ "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
+
+ # Dash mp4 audio
+ "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
+ "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
+ "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
+ "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
+ "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
+ "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
+ "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
+
+ # Dash webm
+ "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
+ "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
+ "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
+ "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
+ "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
+ "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
+ # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
+ "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
+ "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
+ "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+
+ # Dash webm audio
+ "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
+ "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
+
+ # Dash webm audio with opus inside
+ "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
+ "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
+ "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
+
+ # av01 video only formats sometimes served with "unknown" codecs
+ "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
+ "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
+ "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
+ "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
+}
+
+struct VideoPreferences
+ include JSON::Serializable
+
+ property annotations : Bool
+ property autoplay : Bool
+ property comments : Array(String)
+ property continue : Bool
+ property continue_autoplay : Bool
+ property controls : Bool
+ property listen : Bool
+ property local : Bool
+ property preferred_captions : Array(String)
+ property player_style : String
+ property quality : String
+ property quality_dash : String
+ property raw : Bool
+ property region : String?
+ property related_videos : Bool
+ property speed : Float32 | Float64
+ property video_end : Float64 | Int32
+ property video_loop : Bool
+ property extend_desc : Bool
+ property video_start : Float64 | Int32
+ property volume : Int32
+ property vr_mode : Bool
+ property save_player_pos : Bool
end
struct Video
include DB::Serializable
- # Version of the JSON structure
- # It prevents us from loading an incompatible version from cache
- # (either newer or older, if instances with different versions run
- # concurrently, e.g during a version upgrade rollout).
- #
- # NOTE: don't forget to bump this number if any change is made to
- # the `params` structure in videos/parser.cr!!!
- #
- SCHEMA_VERSION = 3
-
property id : String
@[DB::Field(converter: Video::JSONConverter)]
@@ -24,7 +282,13 @@ struct Video
property updated : Time
@[DB::Field(ignore: true)]
- @captions = [] of Invidious::Videos::Captions::Metadata
+ property captions : Array(Caption)?
+
+ @[DB::Field(ignore: true)]
+ property adaptive_fmts : Array(Hash(String, JSON::Any))?
+
+ @[DB::Field(ignore: true)]
+ property fmt_stream : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property description : String?
@@ -35,49 +299,289 @@ struct Video
end
end
- # Methods for API v1 JSON
-
def to_json(locale : String?, json : JSON::Builder)
- Invidious::JSONify::APIv1.video(self, json, locale: locale)
+ json.object do
+ json.field "type", "video"
+
+ json.field "title", self.title
+ json.field "videoId", self.id
+
+ json.field "error", info["reason"] if info["reason"]?
+
+ json.field "videoThumbnails" do
+ generate_thumbnails(json, self.id)
+ end
+ json.field "storyboards" do
+ generate_storyboards(json, self.id, self.storyboards)
+ end
+
+ json.field "description", self.description
+ json.field "descriptionHtml", self.description_html
+ json.field "published", self.published.to_unix
+ json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
+ json.field "keywords", self.keywords
+
+ json.field "viewCount", self.views
+ json.field "likeCount", self.likes
+ json.field "dislikeCount", 0_i64
+
+ json.field "paid", self.paid
+ json.field "premium", self.premium
+ json.field "isFamilyFriendly", self.is_family_friendly
+ json.field "allowedRegions", self.allowed_regions
+ json.field "genre", self.genre
+ json.field "genreUrl", self.genre_url
+
+ json.field "author", self.author
+ json.field "authorId", self.ucid
+ json.field "authorUrl", "/channel/#{self.ucid}"
+
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+
+ json.field "subCountText", self.sub_count_text
+
+ json.field "lengthSeconds", self.length_seconds
+ json.field "allowRatings", self.allow_ratings
+ json.field "rating", 0_i64
+ json.field "isListed", self.is_listed
+ json.field "liveNow", self.live_now
+ json.field "isUpcoming", self.is_upcoming
+
+ if self.premiere_timestamp
+ json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
+ end
+
+ if hlsvp = self.hls_manifest_url
+ hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
+ json.field "hlsUrl", hlsvp
+ end
+
+ json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
+
+ json.field "adaptiveFormats" do
+ json.array do
+ self.adaptive_fmts.each do |fmt|
+ json.object do
+ # Only available on regular videos, not livestreams/OTF streams
+ if init_range = fmt["initRange"]?
+ json.field "init", "#{init_range["start"]}-#{init_range["end"]}"
+ end
+ if index_range = fmt["indexRange"]?
+ json.field "index", "#{index_range["start"]}-#{index_range["end"]}"
+ end
+
+ # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only)
+ json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
+
+ json.field "url", fmt["url"]
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
+ json.field "clen", fmt["contentLength"]? || "-1"
+ json.field "lmt", fmt["lastModified"]
+ json.field "projectionType", fmt["projectionType"]
+
+ if fmt_info = itag_to_metadata?(fmt["itag"])
+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+ json.field "fps", fps
+ json.field "container", fmt_info["ext"]
+ json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
+
+ if fmt_info["height"]?
+ json.field "resolution", "#{fmt_info["height"]}p"
+
+ quality_label = "#{fmt_info["height"]}p"
+ if fps > 30
+ quality_label += "60"
+ end
+ json.field "qualityLabel", quality_label
+
+ if fmt_info["width"]?
+ json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
+ end
+ end
+ end
+
+ # Livestream chunk infos
+ json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec")
+ json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec")
+
+ # Audio-related data
+ json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality")
+ json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate")
+ json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels")
+
+ # Extra misc stuff
+ json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo")
+ json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack")
+ end
+ end
+ end
+ end
+
+ json.field "formatStreams" do
+ json.array do
+ self.fmt_stream.each do |fmt|
+ json.object do
+ json.field "url", fmt["url"]
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
+ json.field "quality", fmt["quality"]
+
+ fmt_info = itag_to_metadata?(fmt["itag"])
+ if fmt_info
+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+ json.field "fps", fps
+ json.field "container", fmt_info["ext"]
+ json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
+
+ if fmt_info["height"]?
+ json.field "resolution", "#{fmt_info["height"]}p"
+
+ quality_label = "#{fmt_info["height"]}p"
+ if fps > 30
+ quality_label += "60"
+ end
+ json.field "qualityLabel", quality_label
+
+ if fmt_info["width"]?
+ json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ json.field "captions" do
+ json.array do
+ self.captions.each do |caption|
+ json.object do
+ json.field "label", caption.name
+ json.field "language_code", caption.language_code
+ json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}"
+ end
+ end
+ end
+ end
+
+ json.field "recommendedVideos" do
+ json.array do
+ self.related_videos.each do |rv|
+ if rv["id"]?
+ json.object do
+ json.field "videoId", rv["id"]
+ json.field "title", rv["title"]
+ json.field "videoThumbnails" do
+ generate_thumbnails(json, rv["id"])
+ end
+
+ json.field "author", rv["author"]
+ json.field "authorUrl", "/channel/#{rv["ucid"]?}"
+ json.field "authorId", rv["ucid"]?
+ if rv["author_thumbnail"]?
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+ end
+
+ json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
+ json.field "viewCountText", rv["short_view_count"]?
+ json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
+ end
+ end
+ end
+ end
+ end
+ end
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
- JSON.build do |json|
- Invidious::JSONify::APIv1.video(self, json, locale: locale)
- end
+ JSON.build { |json| to_json(locale, json) }
end
def to_json(json : JSON::Builder | Nil = nil)
to_json(nil, json)
end
- # Misc methods
-
- def video_type : VideoType
- video_type = info["videoType"]?.try &.as_s || "video"
- return VideoType.parse?(video_type) || VideoType::Video
+ def title
+ info["videoDetails"]["title"]?.try &.as_s || ""
end
- def schema_version : Int
- return info["version"]?.try &.as_i || 1
+ def ucid
+ info["videoDetails"]["channelId"]?.try &.as_s || ""
+ end
+
+ def author
+ info["videoDetails"]["author"]?.try &.as_s || ""
+ end
+
+ def length_seconds : Int32
+ info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i ||
+ info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
+ end
+
+ def views : Int64
+ info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
+ end
+
+ def likes : Int64
+ info["likes"]?.try &.as_i64 || 0_i64
+ end
+
+ def dislikes : Int64
+ info["dislikes"]?.try &.as_i64 || 0_i64
end
def published : Time
- return info["published"]?
+ info
+ .dig?("microformat", "playerMicroformatRenderer", "publishDate")
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
end
def published=(other : Time)
- info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
+ info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
+ end
+
+ def allow_ratings
+ r = info["videoDetails"]["allowRatings"]?.try &.as_bool
+ r.nil? ? false : r
end
def live_now
- return (self.video_type == VideoType::Livestream)
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?
+ .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false
end
- def post_live_dvr
- return info["isPostLiveDvr"].as_bool
+ def is_listed
+ info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
+ end
+
+ def is_upcoming
+ info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
end
def premiere_timestamp : Time?
@@ -86,30 +590,71 @@ struct Video
.try { |t| Time.parse_rfc3339(t.as_s) }
end
+ def keywords
+ info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
+ end
+
def related_videos
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
end
- # Methods for parsing streaming data
-
- def fmt_stream : Array(Hash(String, JSON::Any))
- if formats = info.dig?("streamingData", "formats")
- return formats
- .as_a.map(&.as_h)
- .sort_by! { |f| f["width"]?.try &.as_i || 0 }
- else
- return [] of Hash(String, JSON::Any)
- end
+ def allowed_regions
+ info
+ .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
+ .try &.as_a.map &.as_s || [] of String
end
- def adaptive_fmts : Array(Hash(String, JSON::Any))
- if formats = info.dig?("streamingData", "adaptiveFormats")
- return formats
- .as_a.map(&.as_h)
- .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 }
- else
- return [] of Hash(String, JSON::Any)
+ def author_thumbnail : String
+ info["authorThumbnail"]?.try &.as_s || ""
+ end
+
+ def author_verified : Bool
+ info["authorVerified"]?.try &.as_bool || false
+ end
+
+ def sub_count_text : String
+ info["subCountText"]?.try &.as_s || "-"
+ end
+
+ def fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
+
+ fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
end
+
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @fmt_stream = fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any)))
+ end
+
+ def adaptive_fmts
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
+ fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
+ end
+
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @adaptive_fmts = fmt_stream
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
@@ -120,57 +665,143 @@ struct Video
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
- # Misc. methods
-
def storyboards
- container = info.dig?("storyboards") || JSON::Any.new("{}")
- return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds)
+ storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
+ .try &.as_s.split("|")
+
+ if !storyboards
+ if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
+ return [{
+ url: storyboard.split("#")[0],
+ width: 106,
+ height: 60,
+ count: -1,
+ interval: 5000,
+ storyboard_width: 3,
+ storyboard_height: 3,
+ storyboard_count: -1,
+ }]
+ end
+ end
+
+ items = [] of NamedTuple(
+ url: String,
+ width: Int32,
+ height: Int32,
+ count: Int32,
+ interval: Int32,
+ storyboard_width: Int32,
+ storyboard_height: Int32,
+ storyboard_count: Int32)
+
+ return items if !storyboards
+
+ url = URI.parse(storyboards.shift)
+ params = HTTP::Params.parse(url.query || "")
+
+ storyboards.each_with_index do |sb, i|
+ width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
+ params["sigh"] = sigh
+ url.query = params.to_s
+
+ width = width.to_i
+ height = height.to_i
+ count = count.to_i
+ interval = interval.to_i
+ storyboard_width = storyboard_width.to_i
+ storyboard_height = storyboard_height.to_i
+ storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
+
+ items << {
+ url: url.to_s.sub("$L", i).sub("$N", "M$M"),
+ width: width,
+ height: height,
+ count: count,
+ interval: interval,
+ storyboard_width: storyboard_width,
+ storyboard_height: storyboard_height,
+ storyboard_count: storyboard_count,
+ }
+ end
+
+ items
end
def paid
- return (self.reason || "").includes? "requires payment"
+ reason = info.dig?("playabilityStatus", "reason").try &.as_s || ""
+ return reason.includes? "requires payment"
end
def premium
keywords.includes? "YouTube Red"
end
- def captions : Array(Invidious::Videos::Captions::Metadata)
- if @captions.empty? && @info.has_key?("captions")
- @captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"])
- end
+ def captions : Array(Caption)
+ return @captions.as(Array(Caption)) if @captions
+ captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
+ name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
+ language_code = caption["languageCode"].to_s
+ base_url = caption["baseUrl"].to_s
- return @captions
+ caption = Caption.new(name.to_s, language_code, base_url)
+ caption.name = caption.name.split(" - ")[0]
+ caption
+ end
+ captions ||= [] of Caption
+ @captions = captions
+ return @captions.as(Array(Caption))
+ end
+
+ def description
+ description = info
+ .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText")
+ .try &.as_s || ""
+ end
+
+ # TODO
+ def description=(value : String)
+ @description = value
+ end
+
+ def description_html
+ info["descriptionHtml"]?.try &.as_s || ""
+ end
+
+ def description_html=(value : String)
+ info["descriptionHtml"] = JSON::Any.new(value)
+ end
+
+ def short_description
+ info["shortDescription"]?.try &.as_s? || ""
end
def hls_manifest_url : String?
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
end
- def dash_manifest_url : String?
- raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s
- return nil if raw_dash_url.nil?
+ def dash_manifest_url
+ info.dig?("streamingData", "dashManifestUrl").try &.as_s
+ end
- # Use manifest v5 parameter to reduce file size
- # See https://github.com/iv-org/invidious/issues/4186
- dash_url = URI.parse(raw_dash_url)
- dash_query = dash_url.query || ""
-
- if dash_query.empty?
- dash_url.path = "#{dash_url.path}/mpd_version/5"
- else
- dash_url.query = "#{dash_query}&mpd_version=5"
- end
-
- return dash_url.to_s
+ def genre : String
+ info["genre"]?.try &.as_s || ""
end
def genre_url : String?
- info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
+ info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
end
- def vr? : Bool?
- return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
+ def license : String?
+ info["license"]?.try &.as_s
+ end
+
+ def is_family_friendly : Bool
+ info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false
+ end
+
+ def is_vr : Bool?
+ projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
+ return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
end
def projection_type : String?
@@ -180,118 +811,290 @@ struct Video
def reason : String?
info["reason"]?.try &.as_s
end
+end
- def music : Array(VideoMusic)
- info["music"].as_a.map { |music_json|
- VideoMusic.new(
- music_json["song"].as_s,
- music_json["album"].as_s,
- music_json["artist"].as_s,
- music_json["license"].as_s
- )
- }
+struct Caption
+ property name
+ property language_code
+ property base_url
+
+ getter name : String
+ getter language_code : String
+ getter base_url : String
+
+ setter name
+
+ def initialize(@name, @language_code, @base_url)
+ end
+end
+
+class VideoRedirect < Exception
+ property video_id : String
+
+ def initialize(@video_id)
+ end
+end
+
+# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
+# The former is preferred as it has more videos in it. The second has
+# the same 11 first entries as the compact rendered.
+#
+# TODO: "compactRadioRenderer" (Mix) and
+# TODO: Use a proper struct/class instead of a hacky JSON object
+def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
+ return nil if !related["videoId"]?
+
+ # The compact renderer has video length in seconds, where the end
+ # screen rendered has a full text version ("42:40")
+ length = related["lengthInSeconds"]?.try &.as_i.to_s
+ length ||= related.dig?("lengthText", "simpleText").try do |box|
+ decode_length_seconds(box.as_s).to_s
end
- # Macros defining getters/setters for various types of data
+ # Both have "short", so the "long" option shouldn't be required
+ channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
+ .try &.dig?("runs", 0)
- private macro getset_string(name)
- # Return {{name.stringify}} from `info`
- def {{name.id.underscore}} : String
- return info[{{name.stringify}}]?.try &.as_s || ""
- end
+ author = channel_info.try &.dig?("text")
+ author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
- # Update {{name.stringify}} into `info`
- def {{name.id.underscore}}=(value : String)
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
+ ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
+ # "4,088,033 views", only available on compact renderer
+ # and when video is not a livestream
+ view_count = related.dig?("viewCountText", "simpleText")
+ .try &.as_s.gsub(/\D/, "")
+
+ short_view_count = related.try do |r|
+ HelperExtractors.get_short_view_count(r).to_s
end
- private macro getset_string_array(name)
- # Return {{name.stringify}} from `info`
- def {{name.id.underscore}} : Array(String)
- return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String
- end
+ LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
- # Update {{name.stringify}} into `info`
- def {{name.id.underscore}}=(value : Array(String))
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
+ # TODO: when refactoring video types, make a struct for related videos
+ # or reuse an existing type, if that fits.
+ return {
+ "id" => related["videoId"],
+ "title" => related["title"]["simpleText"],
+ "author" => author || JSON::Any.new(""),
+ "ucid" => JSON::Any.new(ucid || ""),
+ "length_seconds" => JSON::Any.new(length || "0"),
+ "view_count" => JSON::Any.new(view_count || "0"),
+ "short_view_count" => JSON::Any.new(short_view_count || "0"),
+ "author_verified" => JSON::Any.new(author_verified),
+ }
+end
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
+def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil)
+ # Init client config for the API
+ client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
+ if context_screen == "embed"
+ client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
end
- {% for op, type in {i32: Int32, i64: Int64} %}
- private macro getset_{{op}}(name)
- def \{{name.id.underscore}} : {{type}}
- return info[\{{name.stringify}}]?.try &.as_i64.to_{{op}} || 0_{{op}}
+ # Fetch data from the player endpoint
+ player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
+
+ playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
+
+ if playability_status != "OK"
+ subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
+ reason = subreason.try &.[]?("simpleText").try &.as_s
+ reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
+ reason ||= player_response.dig("playabilityStatus", "reason").as_s
+
+ # Stop here if video is not a scheduled livestream
+ if playability_status != "LIVE_STREAM_OFFLINE"
+ return {
+ "reason" => JSON::Any.new(reason),
+ }
+ end
+ elsif video_id != player_response.dig("videoDetails", "videoId")
+ # YouTube may return a different video player response than expected.
+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+ raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)")
+ else
+ reason = nil
+ end
+
+ # Don't fetch the next endpoint if the video is unavailable.
+ if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status)
+ next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
+ player_response = player_response.merge(next_response)
+ end
+
+ params = parse_video_info(video_id, player_response)
+ params["reason"] = JSON::Any.new(reason) if reason
+
+ # Fetch the video streams using an Android client in order to get the decrypted URLs and
+ # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs:
+ # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+ if reason.nil?
+ if context_screen == "embed"
+ client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
+ else
+ client_config.client_type = YoutubeAPI::ClientType::Android
+ end
+ android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
+
+ # Sometimes, the video is available from the web client, but not on Android, so check
+ # that here, and fallback to the streaming data from the web client if needed.
+ # See: https://github.com/iv-org/invidious/issues/2549
+ if video_id != android_player.dig("videoDetails", "videoId")
+ # YouTube may return a different video player response than expected.
+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+ raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (ANDROID client)")
+ elsif android_player["playabilityStatus"]["status"] == "OK"
+ params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("")
+ else
+ params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("")
+ end
+ end
+
+ # TODO: clean that up
+ {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f|
+ params[f] = player_response[f] if player_response[f]?
+ end
+
+ return params
+end
+
+def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
+ # Top level elements
+
+ main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
+
+ raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
+
+ primary_results = main_results.dig?("results", "results", "contents")
+
+ raise BrokenTubeException.new("results") if !primary_results
+
+ video_primary_renderer = primary_results
+ .as_a.find(&.["videoPrimaryInfoRenderer"]?)
+ .try &.["videoPrimaryInfoRenderer"]
+
+ video_secondary_renderer = primary_results
+ .as_a.find(&.["videoSecondaryInfoRenderer"]?)
+ .try &.["videoSecondaryInfoRenderer"]
+
+ raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
+ raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
+
+ # Related videos
+
+ LOGGER.debug("extract_video_info: parsing related videos...")
+
+ related = [] of JSON::Any
+
+ # Parse "compactVideoRenderer" items (under secondary results)
+ secondary_results = main_results
+ .dig?("secondaryResults", "secondaryResults", "results")
+ secondary_results.try &.as_a.each do |element|
+ if item = element["compactVideoRenderer"]?
+ related_video = parse_related_video(item)
+ related << JSON::Any.new(related_video) if related_video
+ end
+ end
+
+ # If nothing was found previously, fall back to end screen renderer
+ if related.empty?
+ # Container for "endScreenVideoRenderer" items
+ player_overlays = player_response.dig?(
+ "playerOverlays", "playerOverlayRenderer",
+ "endScreen", "watchNextEndScreenRenderer", "results"
+ )
+
+ player_overlays.try &.as_a.each do |element|
+ if item = element["endScreenVideoRenderer"]?
+ related_video = parse_related_video(item)
+ related << JSON::Any.new(related_video) if related_video
end
-
- def \{{name.id.underscore}}=(value : Int)
- info[\{{name.stringify}}] = JSON::Any.new(value.to_i64)
- end
-
- \{% if flag?(:debug_macros) %} \{{debug}} \{% end %}
end
- {% end %}
-
- private macro getset_bool(name)
- # Return {{name.stringify}} from `info`
- def {{name.id.underscore}} : Bool
- return info[{{name.stringify}}]?.try &.as_bool || false
- end
-
- # Update {{name.stringify}} into `info`
- def {{name.id.underscore}}=(value : Bool)
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
-
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- # Macro to generate ? and = accessor methods for attributes in `info`
- private macro predicate_bool(method_name, name)
- # Return {{name.stringify}} from `info`
- def {{method_name.id.underscore}}? : Bool
- return info[{{name.stringify}}]?.try &.as_bool || false
- end
+ # Likes
- # Update {{name.stringify}} into `info`
- def {{method_name.id.underscore}}=(value : Bool)
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
+ toplevel_buttons = video_primary_renderer
+ .try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
+ if toplevel_buttons
+ likes_button = toplevel_buttons.as_a
+ .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE")
+ .try &.["toggleButtonRenderer"]
+
+ if likes_button
+ likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?)
+ .try &.dig?("accessibility", "accessibilityData", "label")
+ likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
+
+ LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
+ LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
+ end
end
- # Method definitions, using the macros above
+ # Description
- getset_string author
- getset_string authorThumbnail
- getset_string description
- getset_string descriptionHtml
- getset_string genre
- getset_string genreUcid
- getset_string license
- getset_string shortDescription
- getset_string subCountText
- getset_string title
- getset_string ucid
+ short_description = player_response.dig?("videoDetails", "shortDescription")
- getset_string_array allowedRegions
- getset_string_array keywords
+ description_html = video_secondary_renderer.try &.dig?("description", "runs")
+ .try &.as_a.try { |t| content_to_comment_html(t, video_id) }
- getset_i32 lengthSeconds
- getset_i64 likes
- getset_i64 views
+ # Video metadata
- # TODO: Make predicate_bool the default as to adhere to Crystal conventions
- getset_bool allowRatings
- getset_bool authorVerified
- getset_bool isFamilyFriendly
- getset_bool isListed
- predicate_bool upcoming, isUpcoming
+ metadata = video_secondary_renderer
+ .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
+ .try &.as_a
+
+ genre = player_response.dig?("microformat", "playerMicroformatRenderer", "category")
+ genre_ucid = nil
+ license = nil
+
+ metadata.try &.each do |row|
+ metadata_title = row.dig?("metadataRowRenderer", "title", "simpleText").try &.as_s
+ contents = row.dig?("metadataRowRenderer", "contents", 0)
+
+ if metadata_title == "Category"
+ contents = contents.try &.dig?("runs", 0)
+
+ genre = contents.try &.["text"]?
+ genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId")
+ elsif metadata_title == "License"
+ license = contents.try &.dig?("runs", 0, "text")
+ elsif metadata_title == "Licensed to YouTube by"
+ license = contents.try &.["simpleText"]?
+ end
+ end
+
+ # Author infos
+
+ if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
+ author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
+ author_verified = has_verified_badge?(author_info["badges"]?)
+
+ subs_text = author_info["subscriberCountText"]?
+ .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
+ .try &.as_s.split(" ", 2)[0]
+ end
+
+ # Return data
+
+ params = {
+ "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+ "relatedVideos" => JSON::Any.new(related),
+ "likes" => JSON::Any.new(likes || 0_i64),
+ "dislikes" => JSON::Any.new(0_i64),
+ "descriptionHtml" => JSON::Any.new(description_html || ""),
+ "genre" => JSON::Any.new(genre.try &.as_s || ""),
+ "genreUrl" => JSON::Any.new(nil),
+ "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
+ "license" => JSON::Any.new(license.try &.as_s || ""),
+ "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
+ "authorVerified" => JSON::Any.new(author_verified),
+ "subCountText" => JSON::Any.new(subs_text || "-"),
+ }
+
+ return params
end
def get_video(id, refresh = true, region = nil, force_refresh = false)
@@ -301,8 +1104,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
if (refresh &&
(Time.utc - video.updated > 10.minutes) ||
(video.premiere_timestamp.try &.< Time.utc)) ||
- force_refresh ||
- video.schema_version != Video::SCHEMA_VERSION # cache control
+ force_refresh
begin
video = fetch_video(id, region)
Invidious::Database::Videos.update(video)
@@ -326,12 +1128,31 @@ end
def fetch_video(id, region)
info = extract_video_info(video_id: id)
+ allowed_regions = info
+ .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
+ .try &.as_a.map &.as_s || [] of String
+
+ # Check for region-blocks
+ if info["reason"]?.try &.as_s.includes?("your country")
+ bypass_regions = PROXY_LIST.keys & allowed_regions
+ if !bypass_regions.empty?
+ region = bypass_regions[rand(bypass_regions.size)]
+ region_info = extract_video_info(video_id: id, proxy_region: region)
+ region_info["region"] = JSON::Any.new(region) if region
+ info = region_info if !region_info["reason"]?
+ end
+ end
+
+ # Try to fetch video info using an embedded client
+ if info["reason"]?
+ embed_info = extract_video_info(video_id: id, context_screen: "embed")
+ info = embed_info if !embed_info["reason"]?
+ end
+
if reason = info["reason"]?
if reason == "Video unavailable"
raise NotFoundException.new(reason.as_s || "")
- elsif !reason.as_s.starts_with? "Premieres"
- # dont error when it's a premiere.
- # we already parsed most of the data and display the premiere date
+ else
raise InfoException.new(reason.as_s || "")
end
end
@@ -345,6 +1166,10 @@ def fetch_video(id, region)
return video
end
+def itag_to_metadata?(itag : JSON::Any)
+ return VIDEO_FORMATS[itag.to_s]?
+end
+
def process_continuation(query, plid, id)
continuation = nil
if plid
@@ -359,6 +1184,135 @@ def process_continuation(query, plid, id)
continuation
end
+def process_video_params(query, preferences)
+ annotations = query["iv_load_policy"]?.try &.to_i?
+ autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ comments = query["comments"]?.try &.split(",").map(&.downcase)
+ continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ player_style = query["player_style"]?
+ preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
+ quality = query["quality"]?
+ quality_dash = query["quality_dash"]?
+ region = query["region"]?
+ related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ speed = query["speed"]?.try &.rchop("x").to_f?
+ video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ volume = query["volume"]?.try &.to_i?
+ vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+
+ if preferences
+ # region ||= preferences.region
+ annotations ||= preferences.annotations.to_unsafe
+ autoplay ||= preferences.autoplay.to_unsafe
+ comments ||= preferences.comments
+ continue ||= preferences.continue.to_unsafe
+ continue_autoplay ||= preferences.continue_autoplay.to_unsafe
+ listen ||= preferences.listen.to_unsafe
+ local ||= preferences.local.to_unsafe
+ player_style ||= preferences.player_style
+ preferred_captions ||= preferences.captions
+ quality ||= preferences.quality
+ quality_dash ||= preferences.quality_dash
+ related_videos ||= preferences.related_videos.to_unsafe
+ speed ||= preferences.speed
+ video_loop ||= preferences.video_loop.to_unsafe
+ extend_desc ||= preferences.extend_desc.to_unsafe
+ volume ||= preferences.volume
+ vr_mode ||= preferences.vr_mode.to_unsafe
+ save_player_pos ||= preferences.save_player_pos.to_unsafe
+ end
+
+ annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
+ autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
+ comments ||= CONFIG.default_user_preferences.comments
+ continue ||= CONFIG.default_user_preferences.continue.to_unsafe
+ continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
+ listen ||= CONFIG.default_user_preferences.listen.to_unsafe
+ local ||= CONFIG.default_user_preferences.local.to_unsafe
+ player_style ||= CONFIG.default_user_preferences.player_style
+ preferred_captions ||= CONFIG.default_user_preferences.captions
+ quality ||= CONFIG.default_user_preferences.quality
+ quality_dash ||= CONFIG.default_user_preferences.quality_dash
+ related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
+ speed ||= CONFIG.default_user_preferences.speed
+ video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
+ extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
+ volume ||= CONFIG.default_user_preferences.volume
+ vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
+ save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
+
+ annotations = annotations == 1
+ autoplay = autoplay == 1
+ continue = continue == 1
+ continue_autoplay = continue_autoplay == 1
+ listen = listen == 1
+ local = local == 1
+ related_videos = related_videos == 1
+ video_loop = video_loop == 1
+ extend_desc = extend_desc == 1
+ vr_mode = vr_mode == 1
+ save_player_pos = save_player_pos == 1
+
+ if CONFIG.disabled?("dash") && quality == "dash"
+ quality = "high"
+ end
+
+ if CONFIG.disabled?("local") && local
+ local = false
+ end
+
+ if start = query["t"]? || query["time_continue"]? || query["start"]?
+ video_start = decode_time(start)
+ end
+ video_start ||= 0
+
+ if query["end"]?
+ video_end = decode_time(query["end"])
+ end
+ video_end ||= -1
+
+ raw = query["raw"]?.try &.to_i?
+ raw ||= 0
+ raw = raw == 1
+
+ controls = query["controls"]?.try &.to_i?
+ controls ||= 1
+ controls = controls >= 1
+
+ params = VideoPreferences.new({
+ annotations: annotations,
+ autoplay: autoplay,
+ comments: comments,
+ continue: continue,
+ continue_autoplay: continue_autoplay,
+ controls: controls,
+ listen: listen,
+ local: local,
+ player_style: player_style,
+ preferred_captions: preferred_captions,
+ quality: quality,
+ quality_dash: quality_dash,
+ raw: raw,
+ region: region,
+ related_videos: related_videos,
+ speed: speed,
+ video_end: video_end,
+ video_loop: video_loop,
+ extend_desc: extend_desc,
+ video_start: video_start,
+ volume: volume,
+ vr_mode: vr_mode,
+ save_player_pos: save_player_pos,
+ })
+
+ return params
+end
+
def build_thumbnails(id)
return {
{host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"},
@@ -372,3 +1326,34 @@ def build_thumbnails(id)
{host: HOST_URL, height: 90, width: 120, name: "end", url: "3"},
}
end
+
+def generate_thumbnails(json, id)
+ json.array do
+ build_thumbnails(id).each do |thumbnail|
+ json.object do
+ json.field "quality", thumbnail[:name]
+ json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
+ json.field "width", thumbnail[:width]
+ json.field "height", thumbnail[:height]
+ end
+ end
+ end
+end
+
+def generate_storyboards(json, id, storyboards)
+ json.array do
+ storyboards.each do |storyboard|
+ json.object do
+ json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}"
+ json.field "templateUrl", storyboard[:url]
+ json.field "width", storyboard[:width]
+ json.field "height", storyboard[:height]
+ json.field "count", storyboard[:count]
+ json.field "interval", storyboard[:interval]
+ json.field "storyboardWidth", storyboard[:storyboard_width]
+ json.field "storyboardHeight", storyboard[:storyboard_height]
+ json.field "storyboardCount", storyboard[:storyboard_count]
+ end
+ end
+ end
+end
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
deleted file mode 100644
index c811cfe1..00000000
--- a/src/invidious/videos/caption.cr
+++ /dev/null
@@ -1,224 +0,0 @@
-require "json"
-
-module Invidious::Videos
- module Captions
- struct Metadata
- property name : String
- property language_code : String
- property base_url : String
-
- property auto_generated : Bool
-
- def initialize(@name, @language_code, @base_url, @auto_generated)
- end
-
- # Parse the JSON structure from Youtube
- def self.from_yt_json(container : JSON::Any) : Array(Captions::Metadata)
- caption_tracks = container
- .dig?("playerCaptionsTracklistRenderer", "captionTracks")
- .try &.as_a
-
- captions_list = [] of Captions::Metadata
- return captions_list if caption_tracks.nil?
-
- caption_tracks.each do |caption|
- name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
- name = name.to_s.split(" - ")[0]
-
- language_code = caption["languageCode"].to_s
- base_url = caption["baseUrl"].to_s
-
- auto_generated = (caption["kind"]? == "asr")
-
- captions_list << Captions::Metadata.new(name, language_code, base_url, auto_generated)
- end
-
- return captions_list
- end
-
- def timedtext_to_vtt(timedtext : String, tlang = nil) : String
- # In the future, we could just directly work with the url. This is more of a POC
- cues = [] of XML::Node
- tree = XML.parse(timedtext)
- tree = tree.children.first
-
- tree.children.each do |item|
- if item.name == "body"
- item.children.each do |cue|
- if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n")
- cues << cue
- end
- end
- break
- end
- end
-
- settings_field = {
- "Kind" => "captions",
- "Language" => "#{tlang || @language_code}",
- }
-
- result = WebVTT.build(settings_field) do |vtt|
- cues.each_with_index do |node, i|
- start_time = node["t"].to_f.milliseconds
-
- duration = node["d"]?.try &.to_f.milliseconds
-
- duration ||= start_time
-
- if cues.size > i + 1
- end_time = cues[i + 1]["t"].to_f.milliseconds
- else
- end_time = start_time + duration
- end
-
- text = String.build do |io|
- node.children.each do |s|
- io << s.content
- end
- end
-
- vtt.cue(start_time, end_time, text)
- end
- end
-
- return result
- end
- end
-
- # List of all caption languages available on Youtube.
- LANGUAGES = {
- "",
- "English",
- "English (auto-generated)",
- "English (United Kingdom)",
- "English (United States)",
- "Afrikaans",
- "Albanian",
- "Amharic",
- "Arabic",
- "Armenian",
- "Azerbaijani",
- "Bangla",
- "Basque",
- "Belarusian",
- "Bosnian",
- "Bulgarian",
- "Burmese",
- "Cantonese (Hong Kong)",
- "Catalan",
- "Cebuano",
- "Chinese",
- "Chinese (China)",
- "Chinese (Hong Kong)",
- "Chinese (Simplified)",
- "Chinese (Taiwan)",
- "Chinese (Traditional)",
- "Corsican",
- "Croatian",
- "Czech",
- "Danish",
- "Dutch",
- "Dutch (auto-generated)",
- "Esperanto",
- "Estonian",
- "Filipino",
- "Filipino (auto-generated)",
- "Finnish",
- "French",
- "French (auto-generated)",
- "Galician",
- "Georgian",
- "German",
- "German (auto-generated)",
- "Greek",
- "Gujarati",
- "Haitian Creole",
- "Hausa",
- "Hawaiian",
- "Hebrew",
- "Hindi",
- "Hmong",
- "Hungarian",
- "Icelandic",
- "Igbo",
- "Indonesian",
- "Indonesian (auto-generated)",
- "Interlingue",
- "Irish",
- "Italian",
- "Italian (auto-generated)",
- "Japanese",
- "Japanese (auto-generated)",
- "Javanese",
- "Kannada",
- "Kazakh",
- "Khmer",
- "Korean",
- "Korean (auto-generated)",
- "Kurdish",
- "Kyrgyz",
- "Lao",
- "Latin",
- "Latvian",
- "Lithuanian",
- "Luxembourgish",
- "Macedonian",
- "Malagasy",
- "Malay",
- "Malayalam",
- "Maltese",
- "Maori",
- "Marathi",
- "Mongolian",
- "Nepali",
- "Norwegian Bokmål",
- "Nyanja",
- "Pashto",
- "Persian",
- "Polish",
- "Portuguese",
- "Portuguese (auto-generated)",
- "Portuguese (Brazil)",
- "Punjabi",
- "Romanian",
- "Russian",
- "Russian (auto-generated)",
- "Samoan",
- "Scottish Gaelic",
- "Serbian",
- "Shona",
- "Sindhi",
- "Sinhala",
- "Slovak",
- "Slovenian",
- "Somali",
- "Southern Sotho",
- "Spanish",
- "Spanish (auto-generated)",
- "Spanish (Latin America)",
- "Spanish (Mexico)",
- "Spanish (Spain)",
- "Sundanese",
- "Swahili",
- "Swedish",
- "Tajik",
- "Tamil",
- "Telugu",
- "Thai",
- "Turkish",
- "Turkish (auto-generated)",
- "Ukrainian",
- "Urdu",
- "Uzbek",
- "Vietnamese",
- "Vietnamese (auto-generated)",
- "Welsh",
- "Western Frisian",
- "Xhosa",
- "Yiddish",
- "Yoruba",
- "Zulu",
- }
- end
-end
diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr
deleted file mode 100644
index 29c57182..00000000
--- a/src/invidious/videos/clip.cr
+++ /dev/null
@@ -1,22 +0,0 @@
-require "json"
-
-# returns start_time, end_time and clip_title
-def parse_clip_parameters(params) : {Float64?, Float64?, String?}
- decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
- .try { |i| Base64.decode(i) }
- .try { |i| IO::Memory.new(i) }
- .try { |i| Protodec::Any.parse(i) }
-
- start_time = decoded_protobuf
- .try(&.["50:0:embedded"]["2:1:varint"].as_i64)
- .try { |i| i/1000 }
-
- end_time = decoded_protobuf
- .try(&.["50:0:embedded"]["3:2:varint"].as_i64)
- .try { |i| i/1000 }
-
- clip_title = decoded_protobuf
- .try(&.["50:0:embedded"]["4:3:string"].as_s)
-
- return start_time, end_time, clip_title
-end
diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr
deleted file mode 100644
index 1371bebb..00000000
--- a/src/invidious/videos/description.cr
+++ /dev/null
@@ -1,82 +0,0 @@
-require "json"
-require "uri"
-
-private def copy_string(str : String::Builder, iter : Iterator, count : Int) : Int
- copied = 0
- while copied < count
- cp = iter.next
- break if cp.is_a?(Iterator::Stop)
-
- if cp == 0x26 # Ampersand (&)
- str << "&"
- elsif cp == 0x27 # Single quote (')
- str << "'"
- elsif cp == 0x22 # Double quote (")
- str << """
- elsif cp == 0x3C # Less-than (<)
- str << "<"
- elsif cp == 0x3E # Greater than (>)
- str << ">"
- else
- str << cp.chr
- end
-
- # A codepoint from the SMP counts twice
- copied += 1 if cp > 0xFFFF
- copied += 1
- end
-
- return copied
-end
-
-def parse_description(desc, video_id : String) : String?
- return "" if desc.nil?
-
- content = desc["content"].as_s
- return "" if content.empty?
-
- commands = desc["commandRuns"]?.try &.as_a
- if commands.nil?
- # Slightly faster than HTML.escape, as we're only doing one pass on
- # the string instead of five for the standard library
- return String.build do |str|
- copy_string(str, content.each_codepoint, content.size)
- end
- end
-
- # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints
- # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are
- # automatically decoded by the JSON parser. It means that we need to count
- # copied byte in a special manner, preventing the use of regular string copy.
- iter = content.each_codepoint
-
- index = 0
-
- return String.build do |str|
- commands.each do |command|
- cmd_start = command["startIndex"].as_i
- cmd_length = command["length"].as_i
-
- # Copy the text chunk between this command and the previous if needed.
- length = cmd_start - index
- index += copy_string(str, iter, length)
-
- # We need to copy the command's text using the iterator
- # and the special function defined above.
- cmd_content = String.build(cmd_length) do |str2|
- copy_string(str2, iter, cmd_length)
- end
-
- link = cmd_content
- if on_tap = command.dig?("onTap", "innertubeCommand")
- link = parse_link_endpoint(on_tap, cmd_content, video_id)
- end
- str << link
- index += cmd_length
- end
-
- # Copy the end of the string (past the last command).
- remaining_length = content.size - index
- copy_string(str, iter, remaining_length) if remaining_length > 0
- end
-end
diff --git a/src/invidious/videos/formats.cr b/src/invidious/videos/formats.cr
deleted file mode 100644
index e98e7257..00000000
--- a/src/invidious/videos/formats.cr
+++ /dev/null
@@ -1,116 +0,0 @@
-module Invidious::Videos::Formats
- def self.itag_to_metadata?(itag : JSON::Any)
- return FORMATS[itag.to_s]?
- end
-
- # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
- private FORMATS = {
- "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
- "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
- "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
- "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
- "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
- "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
-
- "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
- "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
-
- # 3D videos
- "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
-
- # Apple HTTP Live Streaming
- "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
- "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
- "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
-
- # DASH mp4 video
- "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
- "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
- "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
- "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
- "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
- "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
- "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
- "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
- "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
- "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
- "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
- "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
-
- # Dash mp4 audio
- "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
- "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
- "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
- "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
- "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
- "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
- "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
-
- # Dash webm
- "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
- "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
- "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
- "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
- "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
- "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
- # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
- "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
- "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
- "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
-
- # Dash webm audio
- "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
- "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
-
- # Dash webm audio with opus inside
- "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
- "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
- "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
-
- # av01 video only formats sometimes served with "unknown" codecs
- "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
- "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
- "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
- "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
- }
-end
diff --git a/src/invidious/videos/music.cr b/src/invidious/videos/music.cr
deleted file mode 100644
index 08d88a3e..00000000
--- a/src/invidious/videos/music.cr
+++ /dev/null
@@ -1,13 +0,0 @@
-require "json"
-
-struct VideoMusic
- include JSON::Serializable
-
- property song : String
- property album : String
- property artist : String
- property license : String
-
- def initialize(@song : String, @album : String, @artist : String, @license : String)
- end
-end
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
deleted file mode 100644
index feb58440..00000000
--- a/src/invidious/videos/parser.cr
+++ /dev/null
@@ -1,504 +0,0 @@
-require "json"
-
-# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
-# The former is preferred as it has more videos in it. The second has
-# the same 11 first entries as the compact rendered.
-#
-# TODO: "compactRadioRenderer" (Mix) and
-# TODO: Use a proper struct/class instead of a hacky JSON object
-def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
- return nil if !related["videoId"]?
-
- # The compact renderer has video length in seconds, where the end
- # screen rendered has a full text version ("42:40")
- length = related["lengthInSeconds"]?.try &.as_i.to_s
- length ||= related.dig?("lengthText", "simpleText").try do |box|
- decode_length_seconds(box.as_s).to_s
- end
-
- # Both have "short", so the "long" option shouldn't be required
- channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
- .try &.dig?("runs", 0)
-
- author = channel_info.try &.dig?("text")
- author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
-
- ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
-
- # "4,088,033 views", only available on compact renderer
- # and when video is not a livestream
- view_count = related.dig?("viewCountText", "simpleText")
- .try &.as_s.gsub(/\D/, "")
-
- short_view_count = related.try do |r|
- HelperExtractors.get_short_view_count(r).to_s
- end
-
- LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
-
- if published_time_text = related["publishedTimeText"]?
- decoded_time = decode_date(published_time_text["simpleText"].to_s)
- published = decoded_time.to_rfc3339.to_s
- else
- published = nil
- end
-
- # TODO: when refactoring video types, make a struct for related videos
- # or reuse an existing type, if that fits.
- return {
- "id" => related["videoId"],
- "title" => related["title"]["simpleText"],
- "author" => author || JSON::Any.new(""),
- "ucid" => JSON::Any.new(ucid || ""),
- "length_seconds" => JSON::Any.new(length || "0"),
- "view_count" => JSON::Any.new(view_count || "0"),
- "short_view_count" => JSON::Any.new(short_view_count || "0"),
- "author_verified" => JSON::Any.new(author_verified),
- "published" => JSON::Any.new(published || ""),
- }
-end
-
-def extract_video_info(video_id : String)
- # Init client config for the API
- client_config = YoutubeAPI::ClientConfig.new
-
- # Fetch data from the player endpoint
- player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
-
- playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
-
- if playability_status != "OK"
- subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
- reason = subreason.try &.[]?("simpleText").try &.as_s
- reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
- reason ||= player_response.dig("playabilityStatus", "reason").as_s
-
- # Stop here if video is not a scheduled livestream or
- # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
- if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
- playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
- return {
- "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
- "reason" => JSON::Any.new(reason),
- }
- end
- elsif video_id != player_response.dig?("videoDetails", "videoId")
- # YouTube may return a different video player response than expected.
- # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
- # Line to be reverted if one day we solve the video not available issue.
-
- # Although technically not a call to /videoplayback the fact that YouTube is returning the
- # wrong video means that we should count it as a failure.
- get_playback_statistic()["totalRequests"] += 1
-
- return {
- "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
- "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."),
- }
- else
- reason = nil
- end
-
- # Don't fetch the next endpoint if the video is unavailable.
- if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
- next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
- player_response = player_response.merge(next_response)
- end
-
- params = parse_video_info(video_id, player_response)
- params["reason"] = JSON::Any.new(reason) if reason
-
- if !CONFIG.invidious_companion.present?
- if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
- players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile}
-
- players_fallback.each do |player_fallback|
- client_config.client_type = player_fallback
-
- next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
-
- if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
- streaming_data = player_response["streamingData"].as_h
- streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
- player_response["streamingData"] = JSON::Any.new(streaming_data)
- break
- end
- rescue InfoException
- next LOGGER.warn("Failed to fetch streams with #{player_fallback}")
- end
- end
-
- # Seems like video page can still render even without playable streams.
- # its better than nothing.
- #
- # # Were we able to find playable video streams?
- # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- # # No :(
- # end
- end
-
- {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
- params[f] = player_response[f] if player_response[f]?
- end
-
- # Convert URLs, if those are present
- if streaming_data = player_response["streamingData"]?
- %w[formats adaptiveFormats].each do |key|
- streaming_data.as_h[key]?.try &.as_a.each do |format|
- format.as_h["url"] = JSON::Any.new(convert_url(format))
- end
- end
-
- params["streamingData"] = streaming_data
- end
-
- # Data structure version, for cache control
- params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
-
- return params
-end
-
-def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
- LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
- response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
-
- playability_status = response["playabilityStatus"]["status"]
- LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
-
- if id != response.dig?("videoDetails", "videoId")
- # YouTube may return a different video player response than expected.
- # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
- raise InfoException.new(
- "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
- )
- elsif playability_status == "OK"
- return response
- else
- return nil
- end
-end
-
-def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
- # Top level elements
-
- main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
-
- raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
-
- # Primary results are not available on Music videos
- # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725
- if primary_results = main_results.dig?("results", "results", "contents")
- video_primary_renderer = primary_results
- .as_a.find(&.["videoPrimaryInfoRenderer"]?)
- .try &.["videoPrimaryInfoRenderer"]
-
- video_secondary_renderer = primary_results
- .as_a.find(&.["videoSecondaryInfoRenderer"]?)
- .try &.["videoSecondaryInfoRenderer"]
-
- raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
- raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
- end
-
- video_details = player_response.dig?("videoDetails")
- if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
- microformat = {} of String => JSON::Any
- end
-
- raise BrokenTubeException.new("videoDetails") if !video_details
-
- # Basic video infos
-
- title = video_details["title"]?.try &.as_s
-
- # We have to try to extract viewCount from videoPrimaryInfoRenderer first,
- # then from videoDetails, as the latter is "0" for livestreams (we want
- # to get the amount of viewers watching).
- views_txt = extract_text(
- video_primary_renderer
- .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount")
- )
- views_txt ||= video_details["viewCount"]?.try &.as_s || ""
- views = views_txt.gsub(/\D/, "").to_i64?
-
- length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
- .try &.as_s.to_i64
-
- published = microformat["publishDate"]?
- .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
-
- premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
- .try { |t| Time.parse_rfc3339(t.as_s) }
-
- premiere_timestamp ||= player_response.dig?(
- "playabilityStatus", "liveStreamability",
- "liveStreamabilityRenderer", "offlineSlate",
- "liveStreamOfflineSlateRenderer", "scheduledStartTime"
- )
- .try &.as_s.to_i64
- .try { |t| Time.unix(t) }
-
- live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
- .try &.as_bool
- live_now ||= video_details.dig?("isLive").try &.as_bool || false
-
- post_live_dvr = video_details.dig?("isPostLiveDvr")
- .try &.as_bool || false
-
- # Extra video infos
-
- allowed_regions = microformat["availableCountries"]?
- .try &.as_a.map &.as_s || [] of String
-
- allow_ratings = video_details["allowRatings"]?.try &.as_bool
- family_friendly = microformat["isFamilySafe"]?.try &.as_bool
- is_listed = video_details["isCrawlable"]?.try &.as_bool
- is_upcoming = video_details["isUpcoming"]?.try &.as_bool
-
- keywords = video_details["keywords"]?
- .try &.as_a.map &.as_s || [] of String
-
- # Related videos
-
- LOGGER.debug("extract_video_info: parsing related videos...")
-
- related = [] of JSON::Any
-
- # Parse "compactVideoRenderer" items (under secondary results)
- secondary_results = main_results
- .dig?("secondaryResults", "secondaryResults", "results")
- secondary_results.try &.as_a.each do |element|
- if item = element["compactVideoRenderer"]?
- related_video = parse_related_video(item)
- related << JSON::Any.new(related_video) if related_video
- end
- end
-
- # If nothing was found previously, fall back to end screen renderer
- if related.empty?
- # Container for "endScreenVideoRenderer" items
- player_overlays = player_response.dig?(
- "playerOverlays", "playerOverlayRenderer",
- "endScreen", "watchNextEndScreenRenderer", "results"
- )
-
- player_overlays.try &.as_a.each do |element|
- if item = element["endScreenVideoRenderer"]?
- related_video = parse_related_video(item)
- related << JSON::Any.new(related_video) if related_video
- end
- end
- end
-
- # Likes
-
- toplevel_buttons = video_primary_renderer
- .try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
-
- if toplevel_buttons
- # New Format as of december 2023
- likes_button = toplevel_buttons.dig?(0,
- "segmentedLikeDislikeButtonViewModel",
- "likeButtonViewModel",
- "likeButtonViewModel",
- "toggleButtonViewModel",
- "toggleButtonViewModel",
- "defaultButtonViewModel",
- "buttonViewModel"
- )
-
- likes_button ||= toplevel_buttons.try &.as_a
- .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE")
- .try &.["toggleButtonRenderer"]
-
- # New format as of september 2022
- likes_button ||= toplevel_buttons.try &.as_a
- .find(&.["segmentedLikeDislikeButtonRenderer"]?)
- .try &.dig?(
- "segmentedLikeDislikeButtonRenderer",
- "likeButton", "toggleButtonRenderer"
- )
-
- if likes_button
- likes_txt = likes_button.dig?("accessibilityText")
- # Note: The like count from `toggledText` is off by one, as it would
- # represent the new like count in the event where the user clicks on "like".
- likes_txt ||= (likes_button["defaultText"]? || likes_button["toggledText"]?)
- .try &.dig?("accessibility", "accessibilityData", "label")
- likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
-
- LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
- LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
- end
- end
-
- # Description
-
- description = microformat.dig?("description", "simpleText").try &.as_s || ""
- short_description = player_response.dig?("videoDetails", "shortDescription")
-
- # description_html = video_secondary_renderer.try &.dig?("description", "runs")
- # .try &.as_a.try { |t| content_to_comment_html(t, video_id) }
-
- description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"), video_id)
-
- # Video metadata
-
- metadata = video_secondary_renderer
- .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
- .try &.as_a
-
- genre = microformat["category"]?
- genre_ucid = nil
- license = nil
-
- metadata.try &.each do |row|
- metadata_title = extract_text(row.dig?("metadataRowRenderer", "title"))
- contents = row.dig?("metadataRowRenderer", "contents", 0)
-
- if metadata_title == "Category"
- contents = contents.try &.dig?("runs", 0)
-
- genre = contents.try &.["text"]?
- genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId")
- elsif metadata_title == "License"
- license = contents.try &.dig?("runs", 0, "text")
- elsif metadata_title == "Licensed to YouTube by"
- license = contents.try &.["simpleText"]?
- end
- end
-
- # Music section
-
- music_list = [] of VideoMusic
- music_desclist = player_response.dig?(
- "engagementPanels", 1, "engagementPanelSectionListRenderer",
- "content", "structuredDescriptionContentRenderer", "items", 2,
- "videoDescriptionMusicSectionRenderer", "carouselLockups"
- )
-
- music_desclist.try &.as_a.each do |music_desc|
- artist = nil
- album = nil
- music_license = nil
-
- # Used when the video has multiple songs
- if song_title = music_desc.dig?("carouselLockupRenderer", "videoLockup", "compactVideoRenderer", "title")
- # "simpleText" for plain text / "runs" when song has a link
- song = song_title["simpleText"]? || song_title.dig?("runs", 0, "text")
-
- # some videos can have empty tracks. See: https://www.youtube.com/watch?v=eBGIQ7ZuuiU
- next if !song
- end
-
- music_desc.dig?("carouselLockupRenderer", "infoRows").try &.as_a.each do |desc|
- desc_title = extract_text(desc.dig?("infoRowRenderer", "title"))
- if desc_title == "ARTIST"
- artist = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
- elsif desc_title == "SONG"
- song = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
- elsif desc_title == "ALBUM"
- album = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
- elsif desc_title == "LICENSES"
- music_license = extract_text(desc.dig?("infoRowRenderer", "expandedMetadata"))
- end
- end
- music_list << VideoMusic.new(song.to_s, album.to_s, artist.to_s, music_license.to_s)
- end
-
- # Author infos
-
- author = video_details["author"]?.try &.as_s
- ucid = video_details["channelId"]?.try &.as_s
-
- if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
- author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
- author_verified = has_verified_badge?(author_info["badges"]?)
-
- subs_text = author_info["subscriberCountText"]?
- .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
- .try &.as_s.split(" ", 2)[0]
- end
-
- # Return data
-
- if live_now
- video_type = VideoType::Livestream
- elsif !premiere_timestamp.nil?
- video_type = VideoType::Scheduled
- published = premiere_timestamp || Time.utc
- else
- video_type = VideoType::Video
- end
-
- params = {
- "videoType" => JSON::Any.new(video_type.to_s),
- # Basic video infos
- "title" => JSON::Any.new(title || ""),
- "views" => JSON::Any.new(views || 0_i64),
- "likes" => JSON::Any.new(likes || 0_i64),
- "lengthSeconds" => JSON::Any.new(length_txt || 0_i64),
- "published" => JSON::Any.new(published.to_rfc3339),
- # Extra video infos
- "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
- "allowRatings" => JSON::Any.new(allow_ratings || false),
- "isFamilyFriendly" => JSON::Any.new(family_friendly || false),
- "isListed" => JSON::Any.new(is_listed || false),
- "isUpcoming" => JSON::Any.new(is_upcoming || false),
- "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
- "isPostLiveDvr" => JSON::Any.new(post_live_dvr),
- # Related videos
- "relatedVideos" => JSON::Any.new(related),
- # Description
- "description" => JSON::Any.new(description || ""),
- "descriptionHtml" => JSON::Any.new(description_html || ""),
- "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
- # Video metadata
- "genre" => JSON::Any.new(genre.try &.as_s || ""),
- "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
- "license" => JSON::Any.new(license.try &.as_s || ""),
- # Music section
- "music" => JSON.parse(music_list.to_json),
- # Author infos
- "author" => JSON::Any.new(author || ""),
- "ucid" => JSON::Any.new(ucid || ""),
- "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
- "authorVerified" => JSON::Any.new(author_verified || false),
- "subCountText" => JSON::Any.new(subs_text || "-"),
- }
-
- return params
-end
-
-private def convert_url(fmt)
- if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
- sp = cfr["sp"]
- url = URI.parse(cfr["url"])
- params = url.query_params
-
- LOGGER.debug("convert_url: Decoding '#{cfr}'")
-
- unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
- params[sp] = unsig if unsig
- else
- url = URI.parse(fmt["url"].as_s)
- params = url.query_params
- end
-
- n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
- params["n"] = n if n
-
- if token = CONFIG.po_token
- params["pot"] = token
- end
-
- url.query_params = params
- LOGGER.trace("convert_url: new url is '#{url}'")
-
- return url.to_s
-rescue ex
- LOGGER.debug("convert_url: Error when parsing video URL")
- LOGGER.trace(ex.inspect_with_backtrace)
- return ""
-end
diff --git a/src/invidious/videos/regions.cr b/src/invidious/videos/regions.cr
deleted file mode 100644
index 575f8c25..00000000
--- a/src/invidious/videos/regions.cr
+++ /dev/null
@@ -1,27 +0,0 @@
-# List of geographical regions that Youtube recognizes.
-# This is used to determine if a video is either restricted to a list
-# of allowed regions (= whitelisted) or if it can't be watched in
-# a set of regions (= blacklisted).
-REGIONS = {
- "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT",
- "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI",
- "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY",
- "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN",
- "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM",
- "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK",
- "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL",
- "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM",
- "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR",
- "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN",
- "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS",
- "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK",
- "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW",
- "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP",
- "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM",
- "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW",
- "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM",
- "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF",
- "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW",
- "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI",
- "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW",
-}
diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr
deleted file mode 100644
index bd0eef59..00000000
--- a/src/invidious/videos/storyboard.cr
+++ /dev/null
@@ -1,122 +0,0 @@
-require "uri"
-require "http/params"
-
-module Invidious::Videos
- struct Storyboard
- # Template URL
- getter url : URI
- getter proxied_url : URI
-
- # Thumbnail parameters
- getter width : Int32
- getter height : Int32
- getter count : Int32
- getter interval : Int32
-
- # Image (storyboard) parameters
- getter rows : Int32
- getter columns : Int32
- getter images_count : Int32
-
- def initialize(
- *, @url, @width, @height, @count, @interval,
- @rows, @columns, @images_count,
- )
- authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]?
-
- @proxied_url = URI.parse(HOST_URL)
- @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}"
- @proxied_url.query = @url.query
- end
-
- # Parse the JSON structure from Youtube
- def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard)
- # Livestream storyboards are a bit different
- # TODO: document exactly how
- if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s
- return [Storyboard.new(
- url: URI.parse(storyboard.split("#")[0]),
- width: 106,
- height: 60,
- count: -1,
- interval: 5000,
- rows: 3,
- columns: 3,
- images_count: -1
- )]
- end
-
- # Split the storyboard string into chunks
- #
- # General format (whitespaces added for legibility):
- # https://i.ytimg.com/sb//storyboard3_L$L/$N.jpg?sqp=
- # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$
- # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$
- # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$
- #
- storyboards = container.dig?("playerStoryboardSpecRenderer", "spec")
- .try &.as_s.split("|")
-
- return [] of Storyboard if !storyboards
-
- # The base URL is the first chunk
- base_url = URI.parse(storyboards.shift)
-
- return storyboards.map_with_index do |sb, i|
- # Separate the different storyboard parameters:
- # width/height: respective dimensions, in pixels, of a single thumbnail
- # count: how many thumbnails are displayed across the full video
- # columns/rows: maximum amount of thumbnails that can be stuffed in a
- # single image, horizontally and vertically.
- # interval: interval between two thumbnails, in milliseconds
- # name: storyboard filename. Usually "M$M" or "default"
- # sigh: URL cryptographic signature
- width, height, count, columns, rows, interval, name, sigh = sb.split("#")
-
- width = width.to_i
- height = height.to_i
- count = count.to_i
- interval = interval.to_i
- columns = columns.to_i
- rows = rows.to_i
-
- # Copy base URL object, so that we can modify it
- url = base_url.dup
-
- # Add the signature to the URL
- params = url.query_params
- params["sigh"] = sigh
- url.query_params = params
-
- # Replace the template parts with what we have
- url.path = url.path.sub("$L", i).sub("$N", name)
-
- # This value represents the maximum amount of thumbnails that can fit
- # in a single image. The last image (or the only one for short videos)
- # will contain less thumbnails than that.
- thumbnails_per_image = columns * rows
-
- # This value represents the total amount of storyboards required to
- # hold all of the thumbnails. It can't be less than 1.
- images_count = (count / thumbnails_per_image).ceil.to_i
-
- # Compute the interval when needed (in general, that's only required
- # for the first "default" storyboard).
- if interval == 0
- interval = ((length_seconds / count) * 1_000).to_i
- end
-
- Storyboard.new(
- url: url,
- width: width,
- height: height,
- count: count,
- interval: interval,
- rows: rows,
- columns: columns,
- images_count: images_count,
- )
- end
- end
- end
-end
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
deleted file mode 100644
index ee1272d1..00000000
--- a/src/invidious/videos/transcript.cr
+++ /dev/null
@@ -1,161 +0,0 @@
-module Invidious::Videos
- # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video.
- # These lines can be categorized into two types: section headings and regular lines representing content from the video.
- struct Transcript
- # Types
- record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- alias TranscriptLine = HeadingLine | RegularLine
-
- property lines : Array(TranscriptLine)
-
- property language_code : String
- property auto_generated : Bool
-
- # User friendly label for the current transcript.
- # Example: "English (auto-generated)"
- property label : String
-
- # Initializes a new Transcript struct with the contents and associated metadata describing it
- def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String)
- end
-
- # Generates a protobuf string to fetch the requested transcript from YouTube
- def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
- kind = auto_generated ? "asr" : ""
-
- object = {
- "1:0:string" => video_id,
-
- "2:base64" => {
- "1:string" => kind,
- "2:string" => language_code,
- "3:string" => "",
- },
-
- "3:varint" => 1_i64,
- "5:string" => "engagement-panel-searchable-transcript-search-panel",
- "6:varint" => 1_i64,
- "7:varint" => 1_i64,
- "8:varint" => 1_i64,
- }
-
- params = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- return params
- end
-
- # Constructs a Transcripts struct from the initial YouTube response
- def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool)
- transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
- "content", "transcriptSearchPanelRenderer")
-
- segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer")
-
- if !segment_list["initialSegments"]?
- raise NotFoundException.new("Requested transcript does not exist")
- end
-
- # Extract user-friendly label for the current transcript
-
- footer_language_menu = transcript_panel.dig?(
- "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems"
- )
-
- if footer_language_menu
- label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
- else
- label = language_code
- end
-
- # Extract transcript lines
-
- initial_segments = segment_list["initialSegments"].as_a
-
- lines = [] of TranscriptLine
-
- initial_segments.each do |line|
- if unpacked_line = line["transcriptSectionHeaderRenderer"]?
- line_type = HeadingLine
- else
- unpacked_line = line["transcriptSegmentRenderer"]
- line_type = RegularLine
- end
-
- start_ms = unpacked_line["startMs"].as_s.to_i.millisecond
- end_ms = unpacked_line["endMs"].as_s.to_i.millisecond
- text = extract_text(unpacked_line["snippet"]) || ""
-
- lines << line_type.new(start_ms, end_ms, text)
- end
-
- return Transcript.new(
- lines: lines,
- language_code: language_code,
- auto_generated: auto_generated,
- label: label
- )
- end
-
- # Converts transcript lines to a WebVTT file
- #
- # This is used within Invidious to replace subtitles
- # as to workaround YouTube's rate-limited timedtext endpoint.
- def to_vtt
- settings_field = {
- "Kind" => "captions",
- "Language" => @language_code,
- }
-
- vtt = WebVTT.build(settings_field) do |builder|
- @lines.each do |line|
- # Section headers are excluded from the VTT conversion as to
- # match the regular captions returned from YouTube as much as possible
- next if line.is_a? HeadingLine
-
- builder.cue(line.start_ms, line.end_ms, line.line)
- end
- end
-
- return vtt
- end
-
- def to_json(json : JSON::Builder)
- json.field "languageCode", @language_code
- json.field "autoGenerated", @auto_generated
- json.field "label", @label
- json.field "body" do
- json.array do
- @lines.each do |line|
- json.object do
- if line.is_a? HeadingLine
- json.field "type", "heading"
- else
- json.field "type", "regular"
- end
-
- json.field "startMs", line.start_ms.total_milliseconds
- json.field "endMs", line.end_ms.total_milliseconds
- json.field "line", line.line
- end
- end
- end
- end
- end
-
- def to_json
- JSON.build do |json|
- json.object do
- json.field "transcript" do
- json.object do
- to_json(json)
- end
- end
- end
- end
- end
- end
-end
diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr
deleted file mode 100644
index 48177bd8..00000000
--- a/src/invidious/videos/video_preferences.cr
+++ /dev/null
@@ -1,162 +0,0 @@
-struct VideoPreferences
- include JSON::Serializable
-
- property annotations : Bool
- property preload : Bool
- property autoplay : Bool
- property comments : Array(String)
- property continue : Bool
- property continue_autoplay : Bool
- property controls : Bool
- property listen : Bool
- property local : Bool
- property preferred_captions : Array(String)
- property player_style : String
- property quality : String
- property quality_dash : String
- property raw : Bool
- property region : String?
- property related_videos : Bool
- property speed : Float32 | Float64
- property video_end : Float64 | Int32
- property video_loop : Bool
- property extend_desc : Bool
- property video_start : Float64 | Int32
- property volume : Int32
- property vr_mode : Bool
- property save_player_pos : Bool
-end
-
-def process_video_params(query, preferences)
- annotations = query["iv_load_policy"]?.try &.to_i?
- preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- comments = query["comments"]?.try &.split(",").map(&.downcase)
- continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- player_style = query["player_style"]?
- preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
- quality = query["quality"]?
- quality_dash = query["quality_dash"]?
- region = query["region"]?
- related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- speed = query["speed"]?.try &.rchop("x").to_f?
- video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- volume = query["volume"]?.try &.to_i?
- vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
-
- if preferences
- # region ||= preferences.region
- annotations ||= preferences.annotations.to_unsafe
- preload ||= preferences.preload.to_unsafe
- autoplay ||= preferences.autoplay.to_unsafe
- comments ||= preferences.comments
- continue ||= preferences.continue.to_unsafe
- continue_autoplay ||= preferences.continue_autoplay.to_unsafe
- listen ||= preferences.listen.to_unsafe
- local ||= preferences.local.to_unsafe
- player_style ||= preferences.player_style
- preferred_captions ||= preferences.captions
- quality ||= preferences.quality
- quality_dash ||= preferences.quality_dash
- related_videos ||= preferences.related_videos.to_unsafe
- speed ||= preferences.speed
- video_loop ||= preferences.video_loop.to_unsafe
- extend_desc ||= preferences.extend_desc.to_unsafe
- volume ||= preferences.volume
- vr_mode ||= preferences.vr_mode.to_unsafe
- save_player_pos ||= preferences.save_player_pos.to_unsafe
- end
-
- annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
- preload ||= CONFIG.default_user_preferences.preload.to_unsafe
- autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
- comments ||= CONFIG.default_user_preferences.comments
- continue ||= CONFIG.default_user_preferences.continue.to_unsafe
- continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
- listen ||= CONFIG.default_user_preferences.listen.to_unsafe
- local ||= CONFIG.default_user_preferences.local.to_unsafe
- player_style ||= CONFIG.default_user_preferences.player_style
- preferred_captions ||= CONFIG.default_user_preferences.captions
- quality ||= CONFIG.default_user_preferences.quality
- quality_dash ||= CONFIG.default_user_preferences.quality_dash
- related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
- speed ||= CONFIG.default_user_preferences.speed
- video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
- extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
- volume ||= CONFIG.default_user_preferences.volume
- vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
- save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
-
- annotations = annotations == 1
- preload = preload == 1
- autoplay = autoplay == 1
- continue = continue == 1
- continue_autoplay = continue_autoplay == 1
- listen = listen == 1
- local = local == 1
- related_videos = related_videos == 1
- video_loop = video_loop == 1
- extend_desc = extend_desc == 1
- vr_mode = vr_mode == 1
- save_player_pos = save_player_pos == 1
-
- if CONFIG.disabled?("dash") && quality == "dash"
- quality = "high"
- end
-
- if CONFIG.disabled?("local") && local
- local = false
- end
-
- if start = query["t"]? || query["time_continue"]? || query["start"]?
- video_start = decode_time(start)
- end
- video_start ||= 0
-
- if query["end"]?
- video_end = decode_time(query["end"])
- end
- video_end ||= -1
-
- raw = query["raw"]?.try &.to_i?
- raw ||= 0
- raw = raw == 1
-
- controls = query["controls"]?.try &.to_i?
- controls ||= 1
- controls = controls >= 1
-
- params = VideoPreferences.new({
- annotations: annotations,
- preload: preload,
- autoplay: autoplay,
- comments: comments,
- continue: continue,
- continue_autoplay: continue_autoplay,
- controls: controls,
- listen: listen,
- local: local,
- player_style: player_style,
- preferred_captions: preferred_captions,
- quality: quality,
- quality_dash: quality_dash,
- raw: raw,
- region: region,
- related_videos: related_videos,
- speed: speed,
- video_end: video_end,
- video_loop: video_loop,
- extend_desc: extend_desc,
- video_start: video_start,
- volume: volume,
- vr_mode: vr_mode,
- save_player_pos: save_player_pos,
- })
-
- return params
-end
diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr
index 6aea82ae..22870317 100644
--- a/src/invidious/views/add_playlist_items.ecr
+++ b/src/invidious/views/add_playlist_items.ecr
@@ -31,5 +31,31 @@
+
+ <% videos.each_slice(4) do |slice| %>
+ <% slice.each do |item| %>
+ <%= rendered "components/item" %>
+ <% end %>
+ <% end %>
+
-<%= rendered "components/items_paginated" %>
+<% if query %>
+ <%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
+
+
+ <% if query.page > 1 %>
+
+ <%= translate(locale, "Previous page") %>
+
+ <% end %>
+
+
+
+ <% if videos.size >= 20 %>
+
+ <%= translate(locale, "Next page") %>
+
+ <% end %>
+
+
+<% end %>
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 686de6bd..92f81ee4 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,59 +1,116 @@
-<%-
- ucid = channel.ucid
- author = HTML.escape(channel.author)
- channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
-
- relative_url =
- case selected_tab
- when .shorts? then "/channel/#{ucid}/shorts"
- when .streams? then "/channel/#{ucid}/streams"
- when .playlists? then "/channel/#{ucid}/playlists"
- when .channels? then "/channel/#{ucid}/channels"
- when .podcasts? then "/channel/#{ucid}/podcasts"
- when .releases? then "/channel/#{ucid}/releases"
- when .courses? then "/channel/#{ucid}/courses"
- else
- "/channel/#{ucid}"
- end
-
- youtube_url = "https://www.youtube.com#{relative_url}"
- redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-
- page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
- base_url: relative_url,
- ctoken: next_continuation,
- first_page: continuation.nil?,
- params: env.params.query,
- )
-%>
+<% ucid = channel.ucid %>
+<% author = HTML.escape(channel.author) %>
<% content_for "header" do %>
-<%- if selected_tab.videos? -%>
-
-
-
-
-
-
-
-
-
-
-
-
-<%- end -%>
-
-
-
-
<%= author %> - Invidious
+
<% end %>
-<%= rendered "components/channel_info" %>
+<% if channel.banner %>
+
+ 
">
+
+
+
+
+
+<% end %>
+
+
+
+
+
+ <%= channel.description_html %>
+
+
+
+
+ <% sub_count_text = number_to_short_text(channel.sub_count) %>
+ <%= rendered "components/subscribe_widget" %>
+
+
+
+
+ <%= translate(locale, "View channel on YouTube") %>
+
+ <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
+ "><%= translate(locale, "Switch Invidious Instance") %>
+ <% else %>
+ <%= translate(locale, "Switch Invidious Instance") %>
+ <% end %>
+
+ <% if !channel.auto_generated %>
+
+ <%= translate(locale, "Videos") %>
+
+ <% end %>
+
+ <% if channel.auto_generated %>
+ <%= translate(locale, "Playlists") %>
+ <% else %>
+ <%= translate(locale, "Playlists") %>
+ <% end %>
+
+
+ <% if channel.tabs.includes? "community" %>
+ <%= translate(locale, "Community") %>
+ <% end %>
+
+
+
+
+
+ <% sort_options.each do |sort| %>
+
+ <% if sort_by == sort %>
+ <%= translate(locale, sort) %>
+ <% else %>
+
+ <%= translate(locale, sort) %>
+
+ <% end %>
+
+ <% end %>
+
+
+
+
+<% items.each do |item| %>
+ <%= rendered "components/item" %>
+<% end %>
+
-<%= rendered "components/items_paginated" %>
+
+
+ <% if page > 1 %>
+ &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <%= translate(locale, "Previous page") %>
+
+ <% end %>
+
+
+
+ <% if count == 60 %>
+ &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <%= translate(locale, "Next page") %>
+
+ <% end %>
+
+
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 132e636c..3bc29e55 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,21 +1,71 @@
-<%-
- ucid = channel.ucid
- author = HTML.escape(channel.author)
- channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
-
- relative_url = "/channel/#{ucid}/community"
- youtube_url = "https://www.youtube.com#{relative_url}"
- redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-
- selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Posts
--%>
+<% ucid = channel.ucid %>
+<% author = HTML.escape(channel.author) %>
<% content_for "header" do %>
-
<%= author %> - Invidious
<% end %>
-<%= rendered "components/channel_info" %>
+<% if channel.banner %>
+
+ ">
+
+
+
+
+
+<% end %>
+
+
+
+
+
+ <%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>
+
+
+
+
+ <% sub_count_text = number_to_short_text(channel.sub_count) %>
+ <%= rendered "components/subscribe_widget" %>
+
+
+
+
+ <%= translate(locale, "View channel on YouTube") %>
+
+ <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
+ "><%= translate(locale, "Switch Invidious Instance") %>
+ <% else %>
+ <%= translate(locale, "Switch Invidious Instance") %>
+ <% end %>
+
+ <% if !channel.auto_generated %>
+
+ <% end %>
+
+
+ <% if channel.tabs.includes? "community" %>
+ <%= translate(locale, "Community") %>
+ <% end %>
+
+
+
+
@@ -26,8 +76,8 @@
<%= error_message %>
<% else %>
-
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 7c584d15..c4eb7507 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -46,14 +46,8 @@ struct PlaylistVideo XML.build { |xml| to_xml(xml) } end - def to_json(locale : String?, json : JSON::Builder) - to_json(json) - end - def to_json(json : JSON::Builder, index : Int32? = nil) json.object do - json.field "type", "video" - json.field "title", self.title json.field "videoId", self.id @@ -62,7 +56,7 @@ struct PlaylistVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do - Invidious::JSONify::APIv1.thumbnails(json, self.id) + generate_thumbnails(json, self.id) end if index @@ -73,7 +67,6 @@ struct PlaylistVideo end json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now end end @@ -96,7 +89,6 @@ struct Playlist property views : Int64 property updated : Time property thumbnail : String? - property subtitle : String? def to_json(offset, json : JSON::Builder, video_id : String? = nil) json.object do @@ -108,7 +100,6 @@ struct Playlist json.field "author", self.author json.field "authorId", self.ucid json.field "authorUrl", "/channel/#{self.ucid}" - json.field "subtitle", self.subtitle json.field "authorThumbnails" do json.array do @@ -270,7 +261,7 @@ end def subscribe_playlist(user, playlist) playlist = InvidiousPlaylist.new({ - title: playlist.title[..150], + title: playlist.title.byte_slice(0, 150), id: playlist.id, author: user.email, description: "", # Max 5000 characters @@ -365,16 +356,12 @@ def fetch_playlist(plid : String) updated = Time.utc video_count = 0 - subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle")) - playlist_info["stats"]?.try &.as_a.each do |stat| text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s next if !text if text.includes? "video" video_count = text.gsub(/\D/, "").to_i? || 0 - elsif text.includes? "episode" - video_count = text.gsub(/\D/, "").to_i? || 0 elsif text.includes? "view" views = text.gsub(/\D/, "").to_i64? || 0_i64 else @@ -410,7 +397,6 @@ def fetch_playlist(plid : String) views: views, updated: updated, thumbnail: thumbnail, - subtitle: subtitle, }) end @@ -432,7 +418,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset end - videos = [] of PlaylistVideo | ProblematicTimelineItem + videos = [] of PlaylistVideo until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count # 100 videos per request @@ -448,7 +434,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) - videos = [] of PlaylistVideo | ProblematicTimelineItem + videos = [] of PlaylistVideo if initial_data["contents"]? tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] @@ -500,14 +486,12 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) index: index, }) end - rescue ex - videos << ProblematicTimelineItem.new(parse_exception: ex) end return videos end -def template_playlist(playlist, listen) +def template_playlist(playlist) html = <<-END_HTML
@@ -521,9 +505,9 @@ def template_playlist(playlist, listen)
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr
index c8db207c..9bb73136 100644
--- a/src/invidious/routes/account.cr
+++ b/src/invidious/routes/account.cr
@@ -42,6 +42,11 @@ module Invidious::Routes::Account
sid = sid.as(String)
token = env.params.body["csrf_token"]?
+ # We don't store passwords for Google accounts
+ if !user.password
+ return error_template(400, "Cannot change password for Google accounts")
+ end
+
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
@@ -49,11 +54,11 @@ module Invidious::Routes::Account
end
password = env.params.body["password"]?
- if password.nil? || password.empty?
+ if !password
return error_template(401, "Password is a required field")
end
- new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
+ new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match")
@@ -198,7 +203,7 @@ module Invidious::Routes::Account
referer = get_referer(env)
if !user
- return env.redirect "/login?referer=#{URI.encode_path_segment(env.request.resource)}"
+ return env.redirect referer
end
user = user.as(User)
@@ -240,7 +245,7 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
+ scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
@@ -257,7 +262,6 @@ module Invidious::Routes::Account
end
query["token"] = access_token
- query["username"] = URI.encode_path_segment(user.email)
url.query = query.to_s
env.redirect url.to_s
@@ -328,9 +332,17 @@ module Invidious::Routes::Account
end
end
- case action = env.params.query["action"]?
- when "revoke_token"
- session = env.params.query["session"]
+ if env.params.query["action_revoke_token"]?
+ action = "action_revoke_token"
+ else
+ return env.redirect referer
+ end
+
+ session = env.params.query["session"]?
+ session ||= ""
+
+ case action
+ when .starts_with? "action_revoke_token"
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
return error_json(400, "Unsupported action #{action}")
diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr
index c27caad7..bfb8a377 100644
--- a/src/invidious/routes/api/manifest.cr
+++ b/src/invidious/routes/api/manifest.cr
@@ -8,17 +8,14 @@ module Invidious::Routes::API::Manifest
id = env.params.url["id"]
region = env.params.query["region"]?
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
- end
-
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin
video = get_video(id, region: region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
haltf env, status_code: 404
rescue ex
@@ -26,27 +23,28 @@ module Invidious::Routes::API::Manifest
end
if dashmpd = video.dash_manifest_url
- response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
+ manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body
- if response.status_code != 200
- haltf env, status_code: response.status_code
- end
+ manifest = manifest.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
+ url = baseurl.lchop("")
+ url = url.rchop(" ")
+
+ if local
+ uri = URI.parse(url)
+ url = "#{uri.request_target}host/#{uri.host}/"
+ end
- # Proxy URLs for video playback on invidious.
- # Other API clients can get the original URLs by omiting `local=true`.
- manifest = response.body.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
- url = baseurl.lchop("").rchop(" ")
- url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
"#{url} "
end
return manifest
end
- # Ditto, only proxify URLs if `local=true` is used
+ adaptive_fmts = video.adaptive_fmts
+
if local
- video.adaptive_fmts.each do |fmt|
- fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
+ adaptive_fmts.each do |fmt|
+ fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target)
end
end
@@ -68,23 +66,17 @@ module Invidious::Routes::API::Manifest
# OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
- audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any
- lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und"
- is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0
- displayname = audio_track["displayName"]?.try &.as_s || "Unknown"
- bitrate = fmt["bitrate"]
-
# Different representations of the same audio should be groupped into one AdaptationSet.
# However, most players don't support auto quality switching, so we have to trick them
# into providing a quality selector.
# See https://github.com/iv-org/invidious/issues/3074 for more details.
- xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do
+ xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
- xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate")
+ xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate")
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
@@ -181,9 +173,8 @@ module Invidious::Routes::API::Manifest
manifest = response.body
if local
- manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match|
- uri = URI.parse(match)
- path = uri.path
+ manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
+ path = URI.parse(match).path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
@@ -212,7 +203,7 @@ module Invidious::Routes::API::Manifest
raw_params["fvip"] = fvip["fvip"]
end
- raw_params["host"] = uri.host.not_nil!
+ raw_params["local"] = "true"
"#{HOST_URL}/videoplayback?#{raw_params}"
end
diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr
index a35d2f2b..1f5ad8ef 100644
--- a/src/invidious/routes/api/v1/authenticated.cr
+++ b/src/invidious/routes/api/v1/authenticated.cr
@@ -31,88 +31,6 @@ module Invidious::Routes::API::V1::Authenticated
env.response.status_code = 204
end
- def self.export_invidious(env)
- env.response.content_type = "application/json"
- user = env.get("user").as(User)
-
- return Invidious::User::Export.to_invidious(user)
- end
-
- def self.import_invidious(env)
- user = env.get("user").as(User)
-
- begin
- if body = env.request.body
- body = env.request.body.not_nil!.gets_to_end
- else
- body = "{}"
- end
- Invidious::User::Import.from_invidious(user, body)
- rescue
- end
-
- env.response.status_code = 204
- end
-
- def self.get_history(env)
- env.response.content_type = "application/json"
- user = env.get("user").as(User)
-
- page = env.params.query["page"]?.try &.to_i?.try &.clamp(0, Int32::MAX)
- page ||= 1
-
- max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
- max_results ||= user.preferences.max_results
- max_results ||= CONFIG.default_user_preferences.max_results
-
- start_index = (page - 1) * max_results
- if user.watched[start_index]?
- watched = user.watched.reverse[start_index, max_results]
- end
- watched ||= [] of String
-
- return watched.to_json
- end
-
- def self.mark_watched(env)
- user = env.get("user").as(User)
-
- if !user.preferences.watch_history
- return error_json(409, "Watch history is disabled in preferences.")
- end
-
- id = env.params.url["id"]
- if !id.match(/^[a-zA-Z0-9_-]{11}$/)
- return error_json(400, "Invalid video id.")
- end
-
- Invidious::Database::Users.mark_watched(user, id)
- env.response.status_code = 204
- end
-
- def self.mark_unwatched(env)
- user = env.get("user").as(User)
-
- if !user.preferences.watch_history
- return error_json(409, "Watch history is disabled in preferences.")
- end
-
- id = env.params.url["id"]
- if !id.match(/^[a-zA-Z0-9_-]{11}$/)
- return error_json(400, "Invalid video id.")
- end
-
- Invidious::Database::Users.mark_unwatched(user, id)
- env.response.status_code = 204
- end
-
- def self.clear_history(env)
- user = env.get("user").as(User)
-
- Invidious::Database::Users.clear_watch_history(user)
- env.response.status_code = 204
- end
-
def self.feed(env)
env.response.content_type = "application/json"
@@ -178,6 +96,10 @@ module Invidious::Routes::API::V1::Authenticated
Invidious::Database::Users.subscribe_channel(user, ucid)
end
+ # For Google accounts, access tokens don't have enough information to
+ # make a request on the user's behalf, which is why we don't sync with
+ # YouTube.
+
env.response.status_code = 204
end
@@ -304,8 +226,8 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(403, "Invalid user")
end
- if playlist.index.size >= CONFIG.playlist_length_limit
- return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
+ if playlist.index.size >= 500
+ return error_json(400, "Playlist cannot have more than 500 videos")
end
video_id = env.params.json["videoId"].try &.as(String)
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index a940ee68..6b81c546 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -1,7 +1,13 @@
module Invidious::Routes::API::V1::Channels
- # Macro to avoid duplicating some code below
- # This sets the `channel` variable, or handles Exceptions.
- private macro get_channel
+ def self.home(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ sort_by = env.params.query["sort_by"]?.try &.downcase
+ sort_by ||= "newest"
+
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
@@ -12,33 +18,14 @@ module Invidious::Routes::API::V1::Channels
rescue ex
return error_json(500, ex)
end
- end
- def self.home(env)
- locale = env.get("preferences").as(Preferences).locale
- ucid = env.params.url["ucid"]
-
- env.response.content_type = "application/json"
-
- # Use the private macro defined above.
- channel = nil # Make the compiler happy
- get_channel()
-
- # Retrieve "sort by" setting from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
-
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
+ page = 1
+ if channel.auto_generated
+ videos = [] of SearchVideo
+ count = 0
else
begin
- videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
+ count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
rescue ex
return error_json(500, ex)
end
@@ -95,15 +82,11 @@ module Invidious::Routes::API::V1::Channels
json.field "joined", channel.joined.to_unix
json.field "autoGenerated", channel.auto_generated
- json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html
json.field "allowedRegions", channel.allowed_regions
- json.field "tabs", channel.tabs
- json.field "tags", channel.tags
- json.field "authorVerified", channel.verified
json.field "latestVideos" do
json.array do
@@ -117,13 +100,31 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
- related_channels, _ = fetch_related_channels(channel)
+ related_channels = fetch_related_channels(channel)
rescue ex
- related_channels = [] of SearchChannel
+ related_channels = [] of AboutRelatedChannel
end
related_channels.each do |related_channel|
- related_channel.to_json(locale, json)
+ json.object do
+ json.field "author", related_channel.author
+ json.field "authorId", related_channel.ucid
+ json.field "authorUrl", related_channel.author_url
+
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+ end
end
end
end # relatedChannels
@@ -133,147 +134,61 @@ module Invidious::Routes::API::V1::Channels
end
def self.latest(env)
- # Remove parameters that could affect this endpoint's behavior
- env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
- env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
+ locale = env.get("preferences").as(Preferences).locale
- return self.videos(env)
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+
+ begin
+ videos = get_latest_videos(ucid)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ JSON.build do |json|
+ json.array do
+ videos.each do |video|
+ video.to_json(locale, json)
+ end
+ end
+ end
end
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
- ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
- # Use the private macro defined above.
- channel = nil # Make the compiler happy
- get_channel()
-
- # Retrieve some URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- continuation = env.params.query["continuation"]?
-
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_60_videos(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
- end
-
- return JSON.build do |json|
- json.object do
- json.field "videos" do
- json.array do
- videos.each &.to_json(locale, json)
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
- def self.shorts(env)
- locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
+ page = env.params.query["page"]?.try &.to_i?
+ page ||= 1
+ sort_by = env.params.query["sort"]?.try &.downcase
+ sort_by ||= env.params.query["sort_by"]?.try &.downcase
+ sort_by ||= "newest"
- env.response.content_type = "application/json"
-
- # Use the private macro defined above.
- channel = nil # Make the compiler happy
- get_channel()
-
- # Retrieve continuation from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- continuation = env.params.query["continuation"]?
-
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_shorts(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ channel = get_about_info(ucid, locale)
+ rescue ex : ChannelRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
+ return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
end
- return JSON.build do |json|
- json.object do
- json.field "videos" do
- json.array do
- videos.each &.to_json(locale, json)
- end
+ begin
+ count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ JSON.build do |json|
+ json.array do
+ videos.each do |video|
+ video.to_json(locale, json)
end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
- def self.streams(env)
- locale = env.get("preferences").as(Preferences).locale
- ucid = env.params.url["ucid"]
-
- env.response.content_type = "application/json"
-
- # Use the private macro defined above.
- channel = nil # Make the compiler happy
- get_channel()
-
- # Retrieve continuation from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- continuation = env.params.query["continuation"]?
-
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_60_livestreams(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
- end
-
- return JSON.build do |json|
- json.object do
- json.field "videos" do
- json.array do
- videos.each &.to_json(locale, json)
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
end
end
end
@@ -289,40 +204,18 @@ module Invidious::Routes::API::V1::Channels
env.params.query["sort_by"]?.try &.downcase ||
"last"
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
-
- JSON.build do |json|
- json.object do
- json.field "playlists" do
- json.array do
- items.each do |item|
- item.to_json(locale, json) if item.is_a?(SearchPlaylist)
- end
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
+ begin
+ channel = get_about_info(ucid, locale)
+ rescue ex : ChannelRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
+ return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
end
- end
- def self.podcasts(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- continuation = env.params.query["continuation"]?
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_podcasts(channel.ucid, channel.author, continuation)
+ items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
JSON.build do |json|
json.object do
@@ -334,65 +227,7 @@ module Invidious::Routes::API::V1::Channels
end
end
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
- def self.releases(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- continuation = env.params.query["continuation"]?
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_releases(channel.ucid, channel.author, continuation)
-
- JSON.build do |json|
- json.object do
- json.field "playlists" do
- json.array do
- items.each do |item|
- item.to_json(locale, json) if item.is_a?(SearchPlaylist)
- end
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
- def self.courses(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- continuation = env.params.query["continuation"]?
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation)
-
- JSON.build do |json|
- json.object do
- json.field "playlists" do
- json.array do
- items.each do |item|
- item.to_json(locale, json) if item.is_a?(SearchPlaylist)
- end
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
+ json.field "continuation", continuation
end
end
end
@@ -420,90 +255,6 @@ module Invidious::Routes::API::V1::Channels
end
end
- def self.post(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
- id = env.params.url["id"].to_s
- ucid = env.params.query["ucid"]?
-
- thin_mode = env.params.query["thin_mode"]?
- thin_mode = thin_mode == "true"
-
- format = env.params.query["format"]?
- format ||= "json"
-
- if ucid.nil?
- response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
- return error_json(400, "Invalid post ID") if response["error"]?
- ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
- else
- ucid = ucid.to_s
- end
-
- begin
- fetch_channel_community_post(ucid, id, locale, format, thin_mode)
- rescue ex
- return error_json(500, ex)
- end
- end
-
- def self.post_comments(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- id = env.params.url["id"]
-
- thin_mode = env.params.query["thin_mode"]?
- thin_mode = thin_mode == "true"
-
- format = env.params.query["format"]?
- format ||= "json"
-
- continuation = env.params.query["continuation"]?
-
- case continuation
- when nil, ""
- ucid = env.params.query["ucid"]
- comments = Comments.fetch_community_post_comments(ucid, id)
- else
- comments = YoutubeAPI.browse(continuation: continuation)
- end
- return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true)
- end
-
- def self.channels(env)
- locale = env.get("preferences").as(Preferences).locale
- ucid = env.params.url["ucid"]
-
- env.response.content_type = "application/json"
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- continuation = env.params.query["continuation"]?
-
- begin
- items, next_continuation = fetch_related_channels(channel, continuation)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.object do
- json.field "relatedChannels" do
- json.array do
- items.each &.to_json(locale, json)
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
def self.search(env)
locale = env.get("preferences").as(Preferences).locale
region = env.params.query["region"]?
diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr
index fea2993c..41865f34 100644
--- a/src/invidious/routes/api/v1/feeds.cr
+++ b/src/invidious/routes/api/v1/feeds.cr
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds
if !CONFIG.popular_enabled
error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
- haltf env, 403, error_message
+ haltf env, 400, error_message
end
JSON.build do |json|
diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr
index 4f5b58da..844fedb8 100644
--- a/src/invidious/routes/api/v1/misc.cr
+++ b/src/invidious/routes/api/v1/misc.cr
@@ -6,22 +6,6 @@ module Invidious::Routes::API::V1::Misc
if !CONFIG.statistics_enabled
return {"software" => SOFTWARE}.to_json
else
- # Calculate playback success rate
- if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
- tracker = tracker.as(Hash(String, Int64 | Float64))
-
- if !tracker.empty?
- total_requests = tracker["totalRequests"]
- success_count = tracker["successfulRequests"]
-
- if total_requests.zero?
- tracker["ratio"] = 1_i64
- else
- tracker["ratio"] = (success_count / (total_requests)).round(2)
- end
- end
- end
-
return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
end
@@ -42,9 +26,6 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
- listen_param = env.params.query["listen"]?
- listen = (listen_param == "true" || listen_param == "1")
-
if plid.starts_with? "RD"
return env.redirect "/api/v1/mixes/#{plid}"
end
@@ -77,9 +58,7 @@ module Invidious::Routes::API::V1::Misc
response = playlist.to_json(offset, video_id: video_id)
json_response = JSON.parse(response)
- if json_response["videos"].as_a.empty?
- json_response = JSON.parse(response)
- elsif json_response["videos"].as_a[0]["index"] != offset
+ if json_response["videos"].as_a[0]["index"] != offset
offset = json_response["videos"].as_a[0]["index"].as_i
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback)
@@ -88,7 +67,7 @@ module Invidious::Routes::API::V1::Misc
end
if format == "html"
- playlist_html = template_playlist(json_response, listen)
+ playlist_html = template_playlist(json_response)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = {
@@ -114,9 +93,6 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
- listen_param = env.params.query["listen"]?
- listen = (listen_param == "true" || listen_param == "1")
-
begin
mix = fetch_mix(rdid, continuation, locale: locale)
@@ -147,7 +123,9 @@ module Invidious::Routes::API::V1::Misc
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "videoThumbnails" do
- Invidious::JSONify::APIv1.thumbnails(json, video.id)
+ json.array do
+ generate_thumbnails(json, video.id)
+ end
end
json.field "index", video.index
@@ -161,7 +139,7 @@ module Invidious::Routes::API::V1::Misc
if format == "html"
response = JSON.parse(response)
- playlist_html = template_mix(response, listen)
+ playlist_html = template_mix(response)
next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"]
response = {
@@ -172,36 +150,4 @@ module Invidious::Routes::API::V1::Misc
response
end
-
- # resolve channel and clip urls, return the UCID
- def self.resolve_url(env)
- env.response.content_type = "application/json"
- url = env.params.query["url"]?
-
- return error_json(400, "Missing URL to resolve") if !url
-
- begin
- resolved_url = YoutubeAPI.resolve_url(url.as(String))
- endpoint = resolved_url["endpoint"]
- page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
- if page_type == "WEB_PAGE_TYPE_UNKNOWN"
- return error_json(400, "Unknown url")
- end
-
- sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
- params = sub_endpoint.try &.dig?("params")
- rescue ex
- return error_json(500, ex)
- end
- JSON.build do |json|
- json.object do
- json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
- json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
- json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
- json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
- json.field "params", params.try &.as_s
- json.field "pageType", page_type
- end
- end
- end
end
diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 59a30745..21451d33 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -31,13 +31,12 @@ module Invidious::Routes::API::V1::Search
query = env.params.query["q"]? || ""
begin
- client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
- url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
+ client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
+ url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&xssi=t&gs_ri=youtube&ds=yt"
response = client.get(url).body
- client.close
- body = JSON.parse(response[19..-2]).as_a
+ body = JSON.parse(response[5..-1]).as_a
suggestions = body[1].as_a[0..-2]
JSON.build do |json|
@@ -56,32 +55,4 @@ module Invidious::Routes::API::V1::Search
return error_json(500, ex)
end
end
-
- def self.hashtag(env)
- hashtag = env.params.url["hashtag"]
-
- page = env.params.query["page"]?.try &.to_i? || 1
-
- locale = env.get("preferences").as(Preferences).locale
- region = env.params.query["region"]?
- env.response.content_type = "application/json"
-
- begin
- results = Invidious::Hashtag.fetch(hashtag, page, region)
- rescue ex
- return error_json(400, ex)
- end
-
- JSON.build do |json|
- json.object do
- json.field "results" do
- json.array do
- results.each do |item|
- item.to_json(locale, json)
- end
- end
- end
- end
- end
- end
end
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index 6a3eb8ae..1b7b4fa7 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -1,5 +1,3 @@
-require "html"
-
module Invidious::Routes::API::V1::Videos
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
@@ -8,19 +6,19 @@ module Invidious::Routes::API::V1::Videos
id = env.params.url["id"]
region = env.params.query["region"]?
- proxy = {"1", "true"}.any? &.== env.params.query["local"]?
begin
video = get_video(id, region: region)
+ rescue ex : VideoRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
+ return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
- return JSON.build do |json|
- Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
- end
+ video.to_json(locale, nil)
end
def self.captions(env)
@@ -43,6 +41,9 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
+ rescue ex : VideoRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
+ return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
haltf env, 404
rescue ex
@@ -89,77 +90,61 @@ module Invidious::Routes::API::V1::Videos
caption = caption[0]
end
- if CONFIG.use_innertube_for_captions
- params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
+ url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
- transcript = Invidious::Videos::Transcript.from_raw(
- YoutubeAPI.get_transcript(params),
- caption.language_code,
- caption.auto_generated
- )
+ # Auto-generated captions often have cues that aren't aligned properly with the video,
+ # as well as some other markup that makes it cumbersome, so we try to fix that here
+ if caption.name.includes? "auto-generated"
+ caption_xml = YT_POOL.client &.get(url).body
+ caption_xml = XML.parse(caption_xml)
- webvtt = transcript.to_vtt
- else
- # Timedtext API handling
- url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
+ webvtt = String.build do |str|
+ str << <<-END_VTT
+ WEBVTT
+ Kind: captions
+ Language: #{tlang || caption.language_code}
- # Auto-generated captions often have cues that aren't aligned properly with the video,
- # as well as some other markup that makes it cumbersome, so we try to fix that here
- if caption.name.includes? "auto-generated"
- caption_xml = YT_POOL.client &.get(url).body
- settings_field = {
- "Kind" => "captions",
- "Language" => "#{tlang || caption.language_code}",
- }
+ END_VTT
- if caption_xml.starts_with?(" i + 1
- end_time = caption_nodes[i + 1]["start"].to_f.seconds
- else
- end_time = start_time + duration
- end
-
- text = HTML.unescape(node.content)
- text = text.gsub(//, "")
- text = text.gsub(/<\/font>/, "")
- if md = text.match(/(?.*) : (?.*)/)
- text = "#{md["text"]} "
- end
-
- builder.cue(start_time, end_time, text)
- end
+ if caption_nodes.size > i + 1
+ end_time = caption_nodes[i + 1]["start"].to_f.seconds
+ else
+ end_time = start_time + duration
end
- end
- else
- uri = URI.parse(url)
- query_params = uri.query_params
- query_params["fmt"] = "vtt"
- uri.query_params = query_params
- webvtt = YT_POOL.client &.get(uri.request_target).body
- if webvtt.starts_with?(" [0-9:.]{12}).+/, "\\1")
+ start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
+ end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
+
+ text = HTML.unescape(node.content)
+ text = text.gsub(//, "")
+ text = text.gsub(/<\/font>/, "")
+ if md = text.match(/(?.*) : (?.*)/)
+ text = "#{md["text"]} "
+ end
+
+ str << <<-END_CUE
+ #{start_time} --> #{end_time}
+ #{text}
+
+
+ END_CUE
end
end
+ else
+ # Some captions have "align:[start/end]" and "position:[num]%"
+ # attributes. Those are causing issues with VideoJS, which is unable
+ # to properly align the captions on the video, so we remove them.
+ #
+ # See: https://github.com/iv-org/invidious/issues/2391
+ webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
+ .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end
if title = env.params.query["title"]?
@@ -183,20 +168,24 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
+ rescue ex : VideoRedirect
+ env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
+ return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
haltf env, 404
rescue ex
haltf env, 500
end
- width = env.params.query["width"]?.try &.to_i
- height = env.params.query["height"]?.try &.to_i
+ storyboards = video.storyboards
+ width = env.params.query["width"]?
+ height = env.params.query["height"]?
if !width && !height
response = JSON.build do |json|
json.object do
json.field "storyboards" do
- Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards)
+ generate_storyboards(json, id, storyboards)
end
end
end
@@ -206,48 +195,43 @@ module Invidious::Routes::API::V1::Videos
env.response.content_type = "text/vtt"
- # Select a storyboard matching the user's provided width/height
- storyboard = video.storyboards.select { |x| x.width == width || x.height == height }
- haltf env, 404 if storyboard.empty?
+ storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" }
- # Alias variable, to make the code below esaier to read
- sb = storyboard[0]
+ if storyboard.empty?
+ haltf env, 404
+ else
+ storyboard = storyboard[0]
+ end
- # Some base URL segments that we'll use to craft the final URLs
- work_url = sb.proxied_url.dup
- template_path = sb.proxied_url.path
+ String.build do |str|
+ str << <<-END_VTT
+ WEBVTT
+ END_VTT
- # Initialize cue timing variables
- # NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap
- # (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000)
- time_delta = sb.interval.milliseconds
- start_time = 0.milliseconds
- end_time = time_delta
+ start_time = 0.milliseconds
+ end_time = storyboard[:interval].milliseconds
- # Build a VTT file for VideoJS-vtt plugin
- vtt_file = WebVTT.build do |vtt|
- sb.images_count.times do |i|
- # Replace the variable component part of the path
- work_url.path = template_path.sub("$M", i)
+ storyboard[:storyboard_count].times do |i|
+ url = storyboard[:url]
+ authority = /(i\d?).ytimg.com/.match(url).not_nil![1]?
+ url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "")
+ url = "#{HOST_URL}/sb/#{authority}/#{url}"
- sb.rows.times do |j|
- sb.columns.times do |k|
- # The URL fragment represents the offset of the thumbnail inside the storyboard image
- work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}"
+ storyboard[:storyboard_height].times do |j|
+ storyboard[:storyboard_width].times do |k|
+ str << <<-END_CUE
+ #{start_time}.000 --> #{end_time}.000
+ #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}
- vtt.cue(start_time, end_time, work_url.to_s)
- start_time += time_delta
- end_time += time_delta
+ END_CUE
+
+ start_time += storyboard[:interval].milliseconds
+ end_time += storyboard[:interval].milliseconds
end
end
end
end
-
- # videojs-vtt-thumbnails is not compliant to the VTT specification, it
- # doesn't unescape the HTML entities, so we have to do it here:
- # TODO: remove this when we migrate to VideoJS 8
- return HTML.unescape(vtt_file)
end
def self.annotations(env)
@@ -268,7 +252,7 @@ module Invidious::Routes::API::V1::Videos
if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
annotations = cached_annotation.annotations
else
- index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
+ index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0')
# IA doesn't handle leading hyphens,
# so we use https://archive.org/details/youtubeannotations_64
@@ -345,7 +329,7 @@ module Invidious::Routes::API::V1::Videos
sort_by ||= "top"
begin
- comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
+ comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
@@ -357,7 +341,7 @@ module Invidious::Routes::API::V1::Videos
sort_by ||= "confidence"
begin
- comments, reddit_thread = Comments.fetch_reddit(id, sort_by: sort_by)
+ comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by)
rescue ex
comments = nil
reddit_thread = nil
@@ -373,9 +357,9 @@ module Invidious::Routes::API::V1::Videos
return reddit_thread.to_json
else
- content_html = Frontend::Comments.template_reddit(comments, locale)
- content_html = Comments.fill_links(content_html, "https", "www.reddit.com")
- content_html = Comments.replace_links(content_html)
+ content_html = template_reddit_comments(comments, locale)
+ content_html = fill_links(content_html, "https", "www.reddit.com")
+ content_html = replace_links(content_html)
response = {
"title" => reddit_thread.title,
"permalink" => reddit_thread.permalink,
@@ -386,133 +370,4 @@ module Invidious::Routes::API::V1::Videos
end
end
end
-
- def self.clips(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- clip_id = env.params.url["id"]
- region = env.params.query["region"]?
- proxy = {"1", "true"}.any? &.== env.params.query["local"]?
-
- response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
- return error_json(400, "Invalid clip ID") if response["error"]?
-
- video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
- return error_json(400, "Invalid clip ID") if video_id.nil?
-
- start_time = nil
- end_time = nil
- clip_title = nil
-
- if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
- start_time, end_time, clip_title = parse_clip_parameters(params)
- end
-
- begin
- video = get_video(video_id, region: region)
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
-
- return JSON.build do |json|
- json.object do
- json.field "startTime", start_time
- json.field "endTime", end_time
- json.field "clipTitle", clip_title
- json.field "video" do
- Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
- end
- end
- end
- end
-
- # Fetches transcripts from YouTube
- #
- # Use the `lang` and `autogen` query parameter to select which transcript to fetch
- # Request without any URL parameters to see all the available transcripts.
- def self.transcripts(env)
- env.response.content_type = "application/json"
-
- id = env.params.url["id"]
- lang = env.params.query["lang"]?
- label = env.params.query["label"]?
- auto_generated = env.params.query["autogen"]? ? true : false
-
- # Return all available transcript options when none is given
- if !label && !lang
- begin
- video = get_video(id)
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
-
- response = JSON.build do |json|
- # The amount of transcripts available to fetch is the
- # same as the amount of captions available.
- available_transcripts = video.captions
-
- json.object do
- json.field "transcripts" do
- json.array do
- available_transcripts.each do |transcript|
- json.object do
- json.field "label", transcript.name
- json.field "languageCode", transcript.language_code
- json.field "autoGenerated", transcript.auto_generated
-
- if transcript.auto_generated
- json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}&autogen"
- else
- json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}"
- end
- end
- end
- end
- end
- end
- end
-
- return response
- end
-
- # If lang is not given then we attempt to fetch
- # the transcript through the given label
- if lang.nil?
- begin
- video = get_video(id)
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
-
- target_transcript = video.captions.select(&.name.== label)
- if target_transcript.empty?
- return error_json(404, NotFoundException.new("Requested transcript does not exist"))
- else
- target_transcript = target_transcript[0]
- lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated
- end
- end
-
- params = Invidious::Videos::Transcript.generate_param(id, lang, auto_generated)
-
- begin
- transcript = Invidious::Videos::Transcript.from_raw(
- YoutubeAPI.get_transcript(params), lang, auto_generated
- )
- rescue ex : NotFoundException
- return error_json(404, ex)
- rescue ex
- return error_json(500, ex)
- end
-
- return transcript.to_json
- end
end
diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr
index b5269668..8e2a253f 100644
--- a/src/invidious/routes/before_all.cr
+++ b/src/invidious/routes/before_all.cr
@@ -20,9 +20,17 @@ module Invidious::Routes::BeforeAll
env.response.headers["X-XSS-Protection"] = "1; mode=block"
env.response.headers["X-Content-Type-Options"] = "nosniff"
+ # Allow media resources to be loaded from google servers
+ # TODO: check if *.youtube.com can be removed
+ if CONFIG.disabled?("local") || !preferences.local
+ extra_media_csp = " https://*.googlevideo.com:443 https://*.youtube.com:443"
+ else
+ extra_media_csp = ""
+ end
+
# Only allow the pages at /embed/* to be embedded
if env.request.resource.starts_with?("/embed")
- frame_ancestors = "'self' file: http: https:"
+ frame_ancestors = "'self' http: https:"
else
frame_ancestors = "'none'"
end
@@ -37,7 +45,7 @@ module Invidious::Routes::BeforeAll
"font-src 'self' data:",
"connect-src 'self'",
"manifest-src 'self'",
- "media-src 'self' blob:",
+ "media-src 'self' blob:" + extra_media_csp,
"child-src 'self' blob:",
"frame-src 'self'",
"frame-ancestors " + frame_ancestors,
@@ -72,23 +80,49 @@ module Invidious::Routes::BeforeAll
raise "Cannot use token as SID"
end
- if email = Database::SessionIDs.select_email(sid)
- user = Database::Users.select!(email: email)
- csrf_token = generate_response(sid, {
- ":authorize_token",
- ":playlist_ajax",
- ":signout",
- ":subscription_ajax",
- ":token_ajax",
- ":watch_ajax",
- }, HMAC_KEY, 1.week)
+ # Invidious users only have SID
+ if !env.request.cookies.has_key? "SSID"
+ if email = Invidious::Database::SessionIDs.select_email(sid)
+ user = Invidious::Database::Users.select!(email: email)
+ csrf_token = generate_response(sid, {
+ ":authorize_token",
+ ":playlist_ajax",
+ ":signout",
+ ":subscription_ajax",
+ ":token_ajax",
+ ":watch_ajax",
+ }, HMAC_KEY, 1.week)
- preferences = user.preferences
- env.set "preferences", preferences
+ preferences = user.preferences
+ env.set "preferences", preferences
- env.set "sid", sid
- env.set "csrf_token", csrf_token
- env.set "user", user
+ env.set "sid", sid
+ env.set "csrf_token", csrf_token
+ env.set "user", user
+ end
+ else
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env.request.headers["Cookie"]
+
+ begin
+ user, sid = get_user(sid, headers, false)
+ csrf_token = generate_response(sid, {
+ ":authorize_token",
+ ":playlist_ajax",
+ ":signout",
+ ":subscription_ajax",
+ ":token_ajax",
+ ":watch_ajax",
+ }, HMAC_KEY, 1.week)
+
+ preferences = user.preferences
+ env.set "preferences", preferences
+
+ env.set "sid", sid
+ env.set "csrf_token", csrf_token
+ env.set "user", user
+ rescue ex
+ end
end
end
@@ -102,21 +136,6 @@ module Invidious::Routes::BeforeAll
preferences.locale = locale
env.set "preferences", preferences
- # Allow media resources to be loaded from google servers
- # TODO: check if *.youtube.com can be removed
- #
- # `!preferences.local` has to be checked after setting and
- # reading `preferences` from the "PREFS" cookie and
- # saved user preferences from the database, otherwise
- # `https://*.googlevideo.com:443 https://*.youtube.com:443`
- # will not be set in the CSP header if
- # `default_user_preferences.local` is set to true on the
- # configuration file, causing preference “Proxy Videos”
- # not to work while having it disabled and using medium quality.
- if CONFIG.disabled?("local") || !preferences.local
- env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"].gsub("media-src", "media-src https://*.googlevideo.com:443 https://*.youtube.com:443")
- end
-
current_page = env.request.path
if env.request.query
query = HTTP::Params.parse(env.request.query.not_nil!)
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index 508aa3e4..c6e02cbd 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -1,32 +1,27 @@
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Channels
- # Redirection for unsupported routes ("tabs")
- def self.redirect_home(env)
- ucid = env.params.url["ucid"]
- return env.redirect "/channel/#{URI.encode_www_form(ucid)}"
- end
-
def self.home(env)
self.videos(env)
end
def self.videos(env)
data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
+ if !data.is_a?(Tuple)
+ return data
+ end
locale, user, subscriptions, continuation, ucid, channel = data
+ page = env.params.query["page"]?.try &.to_i?
+ page ||= 1
+
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
- sort_by ||= "last"
sort_options = {"last", "oldest", "newest"}
+ sort_by ||= "last"
- items, next_continuation = fetch_channel_playlists(
- channel.ucid, channel.author, continuation, sort_by
- )
-
+ items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@@ -34,216 +29,57 @@ module Invidious::Routes::Channels
item.author
end
end
- items = items.select(SearchPlaylist)
+ items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
items.each(&.author = "")
else
- # Fetch items and continuation token
- if channel.is_age_gated
- sort_by = ""
- sort_options = [] of String
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- items = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- items = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- sort_by ||= "newest"
- sort_options = {"newest", "oldest", "popular"}
-
- items, next_continuation = Channel::Tabs.get_60_videos(
- channel, continuation: continuation, sort_by: sort_by
- )
- end
- end
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
- templated "channel"
- end
-
- def self.shorts(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- if !channel.tabs.includes? "shorts"
- return env.redirect "/channel/#{channel.ucid}"
- end
-
- if channel.is_age_gated
- sort_by = ""
- sort_options = [] of String
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
- items = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- items = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
sort_options = {"newest", "oldest", "popular"}
+ sort_by ||= "newest"
- # Fetch items and continuation token
- items, next_continuation = Channel::Tabs.get_shorts(
- channel, continuation: continuation, sort_by: sort_by
- )
+ count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
end
- selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
- templated "channel"
- end
-
- def self.streams(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- if !channel.tabs.includes? "streams"
- return env.redirect "/channel/#{channel.ucid}"
- end
-
- if channel.is_age_gated
- sort_by = ""
- sort_options = [] of String
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
- items = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- items = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- sort_options = {"newest", "oldest", "popular"}
-
- # Fetch items and continuation token
- items, next_continuation = Channel::Tabs.get_60_livestreams(
- channel, continuation: continuation, sort_by: sort_by
- )
- end
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
def self.playlists(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- sort_options = {"last", "oldest", "newest"}
- sort_by = env.params.query["sort_by"]?.try &.downcase
-
- if channel.auto_generated
- return env.redirect "/channel/#{channel.ucid}"
- end
-
- items, next_continuation = fetch_channel_playlists(
- channel.ucid, channel.author, continuation, (sort_by || "last")
- )
-
- items = items.select(SearchPlaylist)
- items.each(&.author = "")
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
- templated "channel"
- end
-
- def self.podcasts(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- sort_by = ""
- sort_options = [] of String
-
- items, next_continuation = fetch_channel_podcasts(
- channel.ucid, channel.author, continuation
- )
-
- items = items.select(SearchPlaylist)
- items.each(&.author = "")
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Podcasts
- templated "channel"
- end
-
- def self.releases(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- sort_by = ""
- sort_options = [] of String
-
- items, next_continuation = fetch_channel_releases(
- channel.ucid, channel.author, continuation
- )
-
- items = items.select(SearchPlaylist)
- items.each(&.author = "")
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Releases
- templated "channel"
- end
-
- def self.courses(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- sort_by = ""
- sort_options = [] of String
-
- items, next_continuation = fetch_channel_courses(
- channel.ucid, channel.author, continuation
- )
-
- items = items.select(SearchPlaylist)
- items.each(&.author = "")
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Courses
- templated "channel"
- end
-
- def self.community(env)
- return env.redirect env.request.path.sub("posts", "community") if env.request.path.split("/").last == "posts"
-
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
locale, user, subscriptions, continuation, ucid, channel = data
- # redirect to post page
- if lb = env.params.query["lb"]?
- env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}"
+ sort_options = {"last", "oldest", "newest"}
+ sort_by = env.params.query["sort_by"]?.try &.downcase
+ sort_by ||= "last"
+
+ if channel.auto_generated
+ return env.redirect "/channel/#{channel.ucid}"
end
+ items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
+ items.each(&.author = "")
+
+ templated "playlists"
+ end
+
+ def self.community(env)
+ data = self.fetch_basic_information(env)
+ if !data.is_a?(Tuple)
+ return data
+ end
+ locale, user, subscriptions, continuation, ucid, channel = data
+
thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
+ # sort_by = env.params.query["sort_by"]?.try &.downcase
- if !channel.tabs.includes? "community" && "posts"
+ if !channel.tabs.includes? "community"
return env.redirect "/channel/#{channel.ucid}"
end
- # TODO: support sort options for community posts
- sort_by = ""
- sort_options = [] of String
-
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
@@ -259,64 +95,6 @@ module Invidious::Routes::Channels
templated "community"
end
- def self.post(env)
- # /post/{postId}
- id = env.params.url["id"]
- ucid = env.params.query["ucid"]?
-
- prefs = env.get("preferences").as(Preferences)
-
- locale = prefs.locale
-
- thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode
- thin_mode = thin_mode == "true"
-
- nojs = env.params.query["nojs"]?
-
- nojs ||= "0"
- nojs = nojs == "1"
-
- if !ucid.nil?
- ucid = ucid.to_s
- post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
- else
- # resolve the url to get the author's UCID
- response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
- return error_template(400, "Invalid post ID") if response["error"]?
-
- ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
- post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
- end
-
- post_response = JSON.parse(post_response)
-
- if nojs
- comments = Comments.fetch_community_post_comments(ucid, id)
- comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"]
- end
- templated "post"
- end
-
- def self.channels(env)
- data = self.fetch_basic_information(env)
- return data if !data.is_a?(Tuple)
-
- locale, user, subscriptions, continuation, ucid, channel = data
-
- if channel.auto_generated
- return env.redirect "/channel/#{channel.ucid}"
- end
-
- items, next_continuation = fetch_related_channels(channel, continuation)
-
- # Featured/related channels can't be sorted
- sort_options = [] of String
- sort_by = nil
-
- selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
- templated "channel"
- end
-
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
@@ -327,12 +105,6 @@ module Invidious::Routes::Channels
env.redirect "/channel/#{ucid}"
end
- private KNOWN_TABS = {
- "home", "videos", "shorts", "streams", "podcasts",
- "releases", "courses", "playlists", "community", "channels", "about",
- "posts",
- }
-
# Redirects brand url channels to a normal /channel/:ucid route
def self.brand_redirect(env)
locale = env.get("preferences").as(Preferences).locale
@@ -343,10 +115,7 @@ module Invidious::Routes::Channels
yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"]))
# Retrieves URL params that only Invidious uses
- invidious_url_params = env.params.query.dup
- invidious_url_params.delete_all("a")
- invidious_url_params.delete_all("u")
- invidious_url_params.delete_all("user")
+ invidious_url_params = URI::Params.encode(env.params.query.to_h.select!(["a", "u", "user"]))
begin
resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}")
@@ -355,17 +124,14 @@ module Invidious::Routes::Channels
return error_template(404, translate(locale, "This channel does not exist."))
end
- selected_tab = env.params.url["tab"]?
-
- if KNOWN_TABS.includes? selected_tab
+ selected_tab = env.request.path.split("/")[-1]
+ if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"
end
- url += "?#{invidious_url_params}" if !invidious_url_params.empty?
-
- return env.redirect url
+ env.redirect url
end
# Handles redirects for the /profile endpoint
@@ -440,7 +206,6 @@ module Invidious::Routes::Channels
return error_template(500, ex)
end
- env.set "search", "channel:#{ucid} "
return {locale, user, subscriptions, continuation, ucid, channel}
end
end
diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr
index 930e4915..84da9993 100644
--- a/src/invidious/routes/embed.cr
+++ b/src/invidious/routes/embed.cr
@@ -2,25 +2,18 @@
module Invidious::Routes::Embed
def self.redirect(env)
- locale = env.get("preferences").as(Preferences).locale
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
- if videos.empty?
- url = "/playlist?list=#{plid}"
- raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
- end
-
- first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{first_playlist_video}?#{env.params.query}"
+ url = "/embed/#{videos[0].id}?#{env.params.query}"
if env.params.query.size > 0
url += "?#{env.params.query}"
@@ -33,7 +26,6 @@ module Invidious::Routes::Embed
end
def self.show(env)
- locale = env.get("preferences").as(Preferences).locale
id = env.params.url["id"]
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
@@ -70,19 +62,13 @@ module Invidious::Routes::Embed
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
- if videos.empty?
- url = "/playlist?list=#{plid}"
- raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
- end
-
- first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{first_playlist_video.id}"
+ url = "/embed/#{videos[0].id}"
elsif video_series
url = "/embed/#{video_series.shift}"
env.params.query["playlist"] = video_series.join(",")
@@ -135,6 +121,8 @@ module Invidious::Routes::Embed
begin
video = get_video(id, region: params.region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
@@ -151,7 +139,7 @@ module Invidious::Routes::Embed
# PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
# end
- if CONFIG.enable_user_notifications && notifications && notifications.includes? id
+ if notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
@@ -161,12 +149,10 @@ module Invidious::Routes::Embed
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
end
- # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -207,14 +193,6 @@ module Invidious::Routes::Embed
return env.redirect url
end
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- env.response.headers["Content-Security-Policy"] =
- env.response.headers["Content-Security-Policy"]
- .gsub("media-src", "media-src #{invidious_companion.public_url}")
- .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
- end
-
rendered "embed"
end
end
diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr
index 1e9ab44e..b138b562 100644
--- a/src/invidious/routes/errors.cr
+++ b/src/invidious/routes/errors.cr
@@ -1,10 +1,5 @@
module Invidious::Routes::ErrorRoutes
def self.error_404(env)
- # Workaround for #3117
- if HOST_URL.empty? && env.request.path.starts_with?("/v1/storyboards/sb")
- return env.redirect "#{env.request.path[15..]}?#{env.params.query}"
- end
-
if md = env.request.path.match(/^\/(?([a-zA-Z0-9_-]{11})|(\w+))$/)
item = md["id"]
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index 070c96eb..b601db94 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -83,6 +83,10 @@ module Invidious::Routes::Feeds
headers = HTTP::Headers.new
headers["Cookie"] = env.request.headers["Cookie"]
+ if !user.password
+ user, sid = get_user(sid, headers)
+ end
+
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
@@ -92,20 +96,14 @@ module Invidious::Routes::Feeds
videos, notifications = get_subscription_feed(user, max_results, page)
- if CONFIG.enable_user_notifications
- # "updated" here is used for delivering new notifications, so if
- # we know a user has looked at their feed e.g. in the past 10 minutes,
- # they've already seen a video posted 20 minutes ago, and don't need
- # to be notified.
- Invidious::Database::Users.clear_notifications(user)
- user.notifications = [] of String
- end
+ # "updated" here is used for delivering new notifications, so if
+ # we know a user has looked at their feed e.g. in the past 10 minutes,
+ # they've already seen a video posted 20 minutes ago, and don't need
+ # to be notified.
+ Invidious::Database::Users.clear_notifications(user)
+ user.notifications = [] of String
env.set "user", user
- # Used for pagination links
- base_url = "/feed/subscriptions"
- base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
-
templated "feeds/subscriptions"
end
@@ -133,91 +131,87 @@ module Invidious::Routes::Feeds
end
watched ||= [] of String
- # Used for pagination links
- base_url = "/feed/history"
- base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
-
templated "feeds/history"
end
# RSS feeds
def self.rss_channel(env)
+ locale = env.get("preferences").as(Preferences).locale
+
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
- if env.params.url["ucid"].matches?(/^[\w-]+$/)
- ucid = env.params.url["ucid"]
- else
- return error_atom(400, InfoException.new("Invalid channel ucid provided."))
- end
+ ucid = env.params.url["ucid"]
params = HTTP::Params.parse(env.params.query["params"]? || "")
- namespaces = {
- "yt" => "http://www.youtube.com/xml/schemas/2015",
- "media" => "http://search.yahoo.com/mrss/",
- "default" => "http://www.w3.org/2005/Atom",
- }
+ begin
+ channel = get_about_info(ucid, locale)
+ rescue ex : ChannelRedirect
+ return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
+ rescue ex : NotFoundException
+ return error_atom(404, ex)
+ rescue ex
+ return error_atom(500, ex)
+ end
- response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
- return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
- rss = XML.parse(response.body)
+ response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
+ rss = XML.parse_html(response.body)
- videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
- video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
- title = entry.xpath_node("default:title", namespaces).not_nil!.content
+ videos = rss.xpath_nodes("//feed/entry").map do |entry|
+ video_id = entry.xpath_node("videoid").not_nil!.content
+ title = entry.xpath_node("title").not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
+ published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
- author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
- video_ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
- description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s
- views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64
+ author = entry.xpath_node("author/name").not_nil!.content
+ ucid = entry.xpath_node("channelid").not_nil!.content
+ description_html = entry.xpath_node("group/description").not_nil!.to_s
+ views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
SearchVideo.new({
title: title,
id: video_id,
author: author,
- ucid: video_ucid,
+ ucid: ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
+ live_now: false,
+ paid: false,
+ premium: false,
premiere_timestamp: nil,
author_verified: false,
- author_thumbnail: nil,
- badges: VideoBadges::None,
})
end
- author = ""
- author = videos[0].author if videos.size > 0
-
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
- xml.element("id") { xml.text "yt:channel:#{ucid}" }
- xml.element("yt:channelId") { xml.text ucid }
- xml.element("title") { xml.text author }
- xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{ucid}")
+ xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
+ xml.element("yt:channelId") { xml.text channel.ucid }
+ xml.element("icon") { xml.text channel.author_thumbnail }
+ xml.element("title") { xml.text channel.author }
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
xml.element("author") do
- xml.element("name") { xml.text author }
- xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
+ xml.element("name") { xml.text channel.author }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
end
xml.element("image") do
- xml.element("url") { xml.text "" }
- xml.element("title") { xml.text author }
+ xml.element("url") { xml.text channel.author_thumbnail }
+ xml.element("title") { xml.text channel.author }
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
end
videos.each do |video|
- video.to_xml(false, params, xml)
+ video.to_xml(channel.auto_generated, params, xml)
end
end
end
@@ -296,13 +290,7 @@ module Invidious::Routes::Feeds
xml.element("name") { xml.text playlist.author }
end
- videos.each do |video|
- if video.is_a? PlaylistVideo
- video.to_xml(xml)
- else
- video.to_xml(env, locale, xml)
- end
- end
+ videos.each &.to_xml(xml)
end
end
else
@@ -311,9 +299,8 @@ module Invidious::Routes::Feeds
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
- return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
-
document = XML.parse(response.body)
+
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
node.attributes.each do |attribute|
case attribute.name
@@ -408,23 +395,22 @@ module Invidious::Routes::Feeds
end
spawn do
- # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow?
- namespaces = {
- "yt" => "http://www.youtube.com/xml/schemas/2015",
- "default" => "http://www.w3.org/2005/Atom",
- }
- rss = XML.parse(body)
- rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
- id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
- author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
+ rss = XML.parse_html(body)
+ rss.xpath_nodes("//feed/entry").each do |entry|
+ id = entry.xpath_node("videoid").not_nil!.content
+ author = entry.xpath_node("author/name").not_nil!.content
+ published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
- begin
- video = get_video(id, force_refresh: true)
- rescue
- next # skip this video since it raised an exception (e.g. it is a scheduled live event)
- end
+ video = get_video(id, force_refresh: true)
+
+ # Deliver notifications to `/api/v1/auth/notifications`
+ payload = {
+ "topic" => video.ucid,
+ "videoId" => video.id,
+ "published" => published.to_unix,
+ }.to_json
+ PG_DB.exec("NOTIFY notifications, E'#{payload}'")
video = ChannelVideo.new({
id: id,
@@ -440,9 +426,7 @@ module Invidious::Routes::Feeds
})
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
- if was_insert
- NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
- end
+ Invidious::Database::Users.add_notification(video) if was_insert
end
end
diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 51d85dfe..594a7869 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -3,7 +3,17 @@ module Invidious::Routes::Images
def self.ggpht(env)
url = env.request.path.lchop("/ggpht")
- headers = HTTP::Headers.new
+ headers = (
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ HTTP::Headers{":authority" => "yt3.ggpht.com"}
+ else
+ HTTP::Headers.new
+ end
+ {% else %}
+ HTTP::Headers.new
+ {% end %}
+ )
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
@@ -11,10 +21,43 @@ module Invidious::Routes::Images
end
end
- begin
- GGPHT_POOL.client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ # We're encapsulating this into a proc in order to easily reuse this
+ # portion of the code for each request block below.
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ env.response.headers.delete("Transfer-Encoding")
+ return
+ end
+
+ proxy_file(response, env)
+ }
+
+ begin
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ YT_POOL.client &.get(url, headers) do |resp|
+ return request_proc.call(resp)
+ end
+ else
+ HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ {% end %}
rescue ex
end
end
@@ -35,17 +78,51 @@ module Invidious::Routes::Images
headers = HTTP::Headers.new
+ {% unless flag?(:disable_quic) %}
+ headers[":authority"] = "#{authority}.ytimg.com"
+ {% end %}
+
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
- begin
- get_ytimg_pool(authority).client &.get(url, headers) do |resp|
- env.response.headers["Connection"] = "close"
- return self.proxy_image(env, resp)
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
end
+
+ env.response.headers["Connection"] = "close"
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
+ begin
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ YT_POOL.client &.get(url, headers) do |resp|
+ return request_proc.call(resp)
+ end
+ else
+ HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ {% end %}
rescue ex
end
end
@@ -56,7 +133,17 @@ module Invidious::Routes::Images
name = env.params.url["name"]
url = env.request.resource
- headers = HTTP::Headers.new
+ headers = (
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ HTTP::Headers{":authority" => "i9.ytimg.com"}
+ else
+ HTTP::Headers.new
+ end
+ {% else %}
+ HTTP::Headers.new
+ {% end %}
+ )
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
@@ -64,10 +151,40 @@ module Invidious::Routes::Images
end
end
- begin
- get_ytimg_pool("i9").client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300 && response.status_code != 404
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
+ begin
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ YT_POOL.client &.get(url, headers) do |resp|
+ return request_proc.call(resp)
+ end
+ else
+ HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ {% end %}
rescue ex
end
end
@@ -106,15 +223,41 @@ module Invidious::Routes::Images
id = env.params.url["id"]
name = env.params.url["name"]
- headers = HTTP::Headers.new
+ headers = (
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ HTTP::Headers{":authority" => "i.ytimg.com"}
+ else
+ HTTP::Headers.new
+ end
+ {% else %}
+ HTTP::Headers.new
+ {% end %}
+ )
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
- if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
- name = thumb[:url] + ".jpg"
- break
- end
+ # Logic here is short enough that manually typing them out should be fine.
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
+ name = thumb[:url] + ".jpg"
+ break
+ end
+ else
+ if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+ name = thumb[:url] + ".jpg"
+ break
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+ name = thumb[:url] + ".jpg"
+ break
+ end
+ {% end %}
end
end
@@ -126,28 +269,41 @@ module Invidious::Routes::Images
end
end
- begin
- get_ytimg_pool("i").client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300 && response.status_code != 404
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
+ begin
+ {% unless flag?(:disable_quic) %}
+ if CONFIG.use_quic
+ YT_POOL.client &.get(url, headers) do |resp|
+ return request_proc.call(resp)
+ end
+ else
+ HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ end
+ {% else %}
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
+ end
+ {% end %}
rescue ex
end
end
-
- private def self.proxy_image(env, response)
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- return proxy_file(response, env)
- end
end
diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr
index e7de5018..99fc13a2 100644
--- a/src/invidious/routes/login.cr
+++ b/src/invidious/routes/login.cr
@@ -6,14 +6,14 @@ module Invidious::Routes::Login
user = env.get? "user"
- referer = get_referer(env, "/feed/subscriptions")
-
- return env.redirect referer if user
+ return env.redirect "/feed/subscriptions" if user
if !CONFIG.login_enabled
return error_template(400, "Login has been disabled by administrator.")
end
+ referer = get_referer(env, "/feed/subscriptions")
+
email = nil
password = nil
captcha = nil
@@ -21,6 +21,12 @@ module Invidious::Routes::Login
account_type = env.params.query["type"]?
account_type ||= "invidious"
+ captcha_type = env.params.query["captcha"]?
+ captcha_type ||= "image"
+
+ tfa = env.params.query["tfa"]?
+ prompt = nil
+
templated "user/login"
end
@@ -41,18 +47,283 @@ module Invidious::Routes::Login
account_type ||= "invidious"
case account_type
+ when "google"
+ tfa_code = env.params.body["tfa"]?.try &.lchop("G-")
+ traceback = IO::Memory.new
+
+ # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
+ begin
+ client = nil # Declare variable
+ {% unless flag?(:disable_quic) %}
+ client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL)
+ {% else %}
+ client = HTTP::Client.new(LOGIN_URL)
+ {% end %}
+
+ headers = HTTP::Headers.new
+
+ login_page = client.get("/ServiceLogin")
+ headers = login_page.cookies.add_request_headers(headers)
+
+ lookup_req = {
+ email, nil, [] of String, nil, "US", nil, nil, 2, false, true,
+ {nil, nil,
+ {2, 1, nil, 1,
+ "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
+ nil, [] of String, 4},
+ 1,
+ {nil, nil, [] of String},
+ nil, nil, nil, true,
+ },
+ email,
+ }.to_json
+
+ traceback << "Getting lookup..."
+
+ headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
+ headers["Google-Accounts-XSRF"] = "1"
+
+ response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
+ lookup_results = JSON.parse(response.body[5..-1])
+
+ traceback << "done, returned #{response.status_code}.
"
+
+ user_hash = lookup_results[0][2]
+
+ if token = env.params.body["token"]?
+ answer = env.params.body["answer"]?
+ captcha = {token, answer}
+ else
+ captcha = nil
+ end
+
+ challenge_req = {
+ user_hash, nil, 1, nil,
+ {1, nil, nil, nil,
+ {password, captcha, true},
+ },
+ {nil, nil,
+ {2, 1, nil, 1,
+ "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
+ nil, [] of String, 4},
+ 1,
+ {nil, nil, [] of String},
+ nil, nil, nil, true,
+ },
+ }.to_json
+
+ traceback << "Getting challenge..."
+
+ response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req))
+ headers = response.cookies.add_request_headers(headers)
+ challenge_results = JSON.parse(response.body[5..-1])
+
+ traceback << "done, returned #{response.status_code}.
"
+
+ headers["Cookie"] = URI.decode_www_form(headers["Cookie"])
+
+ if challenge_results[0][3]?.try &.== 7
+ return error_template(423, "Account has temporarily been disabled")
+ end
+
+ if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s
+ account_type = "google"
+ captcha_type = "image"
+ prompt = nil
+ tfa = tfa_code
+ captcha = {tokens: [token], question: ""}
+
+ return templated "user/login"
+ end
+
+ if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED"
+ return error_template(401, "Incorrect password")
+ end
+
+ prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]?
+ if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type
+ traceback << "Handling prompt #{prompt_type}.
"
+ case prompt_type
+ when "TWO_STEP_VERIFICATION"
+ prompt_type = 2
+ else # "LOGIN_CHALLENGE"
+ prompt_type = 4
+ end
+
+ # Prefer Authenticator app and SMS over unsupported protocols
+ if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2
+ tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0]
+
+ traceback << "Selecting challenge #{tfa[8]}..."
+ select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json
+
+ tl = challenge_results[1][2]
+
+ tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body
+ tfa = tfa[5..-1]
+ tfa = JSON.parse(tfa)[0][-1]
+
+ traceback << "done.
"
+ else
+ traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.
"
+ tfa = challenge_results[0][-1][0][0]
+ end
+
+ if tfa[5] == "QUOTA_EXCEEDED"
+ return error_template(423, "Quota exceeded, try again in a few hours")
+ end
+
+ if !tfa_code
+ account_type = "google"
+ captcha_type = "image"
+
+ case tfa[8]
+ when 6, 9
+ prompt = "Google verification code"
+ when 12
+ prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
+ when 15
+ prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
+ else
+ prompt = "Google verification code"
+ end
+
+ tfa = nil
+ captcha = nil
+ return templated "user/login"
+ end
+
+ tl = challenge_results[1][2]
+
+ request_type = tfa[8]
+ case request_type
+ when 6 # Authenticator app
+ tfa_req = {
+ user_hash, nil, 2, nil,
+ {6, nil, nil, nil, nil,
+ {tfa_code, false},
+ },
+ }.to_json
+ when 9 # Voice or text message
+ tfa_req = {
+ user_hash, nil, 2, nil,
+ {9, nil, nil, nil, nil, nil, nil, nil,
+ {nil, tfa_code, false, 2},
+ },
+ }.to_json
+ when 12 # Recovery email
+ tfa_req = {
+ user_hash, nil, 4, nil,
+ {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+ {tfa_code},
+ },
+ }.to_json
+ when 15 # Security question
+ tfa_req = {
+ user_hash, nil, 5, nil,
+ {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+ {tfa_code},
+ },
+ }.to_json
+ else
+ return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.")
+ end
+
+ traceback << "Submitting challenge..."
+
+ response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req))
+ headers = response.cookies.add_request_headers(headers)
+ challenge_results = JSON.parse(response.body[5..-1])
+
+ if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") ||
+ (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT")
+ return error_template(401, "Invalid TFA code")
+ end
+
+ traceback << "done.
"
+ end
+
+ traceback << "Logging in..."
+
+ location = URI.parse(challenge_results[0][-1][2].to_s)
+ cookies = HTTP::Cookies.from_client_headers(headers)
+
+ headers.delete("Content-Type")
+ headers.delete("Google-Accounts-XSRF")
+
+ loop do
+ if !location || location.path == "/ManageAccount"
+ break
+ end
+
+ # Occasionally there will be a second page after login confirming
+ # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle.
+
+ if location.path.starts_with? "/b/0/SmsAuthInterstitial"
+ traceback << "Unhandled dialog /b/0/SmsAuthInterstitial."
+ end
+
+ login = client.get(location.request_target, headers)
+
+ headers = login.cookies.add_request_headers(headers)
+ location = login.headers["Location"]?.try { |u| URI.parse(u) }
+ end
+
+ cookies = HTTP::Cookies.from_client_headers(headers)
+ sid = cookies["SID"]?.try &.value
+ if !sid
+ raise "Couldn't get SID."
+ end
+
+ user, sid = get_user(sid, headers)
+
+ # We are now logged in
+ traceback << "done.
"
+
+ host = URI.parse(env.request.headers["Host"]).host
+
+ cookies.each do |cookie|
+ cookie.secure = Invidious::User::Cookies::SECURE
+
+ if cookie.extension
+ cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host)
+ cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "")
+ end
+ env.response.cookies << cookie
+ end
+
+ if env.request.cookies["PREFS"]?
+ user.preferences = env.get("preferences").as(Preferences)
+ Invidious::Database::Users.update_preferences(user)
+
+ cookie = env.request.cookies["PREFS"]
+ cookie.expires = Time.utc(1990, 1, 1)
+ env.response.cookies << cookie
+ end
+
+ env.redirect referer
+ rescue ex
+ traceback.rewind
+ # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.")
+ error_message = %(#{ex.message}
Traceback:
#{traceback.gets_to_end})
+ return error_template(500, error_message)
+ end
when "invidious"
- if email.nil? || email.empty?
+ if !email
return error_template(401, "User ID is a required field")
end
- if password.nil? || password.empty?
+ if !password
return error_template(401, "Password is a required field")
end
user = Invidious::Database::Users.select(email: email)
if user
+ if !user.password
+ return error_template(400, "Please sign in using 'Log in with Google'")
+ end
+
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
Invidious::Database::SessionIDs.insert(sid, email)
@@ -85,14 +356,36 @@ module Invidious::Routes::Login
password = password.byte_slice(0, 55)
if CONFIG.captcha_enabled
+ captcha_type = env.params.body["captcha_type"]?
answer = env.params.body["answer"]?
+ change_type = env.params.body["change_type"]?
- account_type = "invidious"
- captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ if !captcha_type || change_type
+ if change_type
+ captcha_type = change_type
+ end
+ captcha_type ||= "image"
+
+ account_type = "invidious"
+ tfa = false
+ prompt = ""
+
+ if captcha_type == "image"
+ captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ else
+ captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
+ end
+
+ return templated "user/login"
+ end
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
- if answer
+ answer ||= ""
+ captcha_type ||= "image"
+
+ case captcha_type
+ when "image"
answer = answer.lstrip('0')
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
@@ -101,8 +394,27 @@ module Invidious::Routes::Login
rescue ex
return error_template(400, ex)
end
- else
- return templated "user/login"
+ else # "text"
+ answer = Digest::MD5.hexdigest(answer.downcase.strip)
+
+ if tokens.empty?
+ return error_template(500, "Erroneous CAPTCHA")
+ end
+
+ found_valid_captcha = false
+ error_exception = Exception.new
+ tokens.each do |tok|
+ begin
+ validate_request(tok, answer, env.request, HMAC_KEY, locale)
+ found_valid_captcha = true
+ rescue ex
+ error_exception = ex
+ end
+ end
+
+ if !found_valid_captcha
+ return error_template(500, error_exception)
+ end
end
end
@@ -169,4 +481,11 @@ module Invidious::Routes::Login
env.redirect referer
end
+
+ def self.captcha(env)
+ headers = HTTP::Headers{":authority" => "accounts.google.com"}
+ response = YT_POOL.client &.get(env.request.resource, headers)
+ env.response.headers["Content-Type"] = response.headers["Content-Type"]
+ response.body
+ end
end
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index 0b868755..d6bd9571 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -40,21 +40,7 @@ module Invidious::Routes::Misc
def self.cross_instance_redirect(env)
referer = get_referer(env)
-
- instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
- # Filter out the current instance
- other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
-
- if other_available_instances.empty?
- # If the current instance is the only one, use the redirect URL as fallback
- instance_url = "redirect.invidious.io"
- else
- # Select other random instance
- # Sample returns an array
- # Instances are packaged as {region, domain} in the instance list
- instance_url = other_available_instances.sample(1)[0][1]
- end
-
+ instance_url = fetch_random_instance
env.redirect "https://#{instance_url}#{referer}"
end
end
diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr
index 8922b740..272a3dc7 100644
--- a/src/invidious/routes/notifications.cr
+++ b/src/invidious/routes/notifications.cr
@@ -24,6 +24,50 @@ module Invidious::Routes::Notifications
user = user.as(User)
+ if !user.password
+ channel_req = {} of String => String
+
+ channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true"
+ channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || ""
+ channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true"
+
+ channel_req.reject! { |k, v| v != "true" && v != "false" }
+
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env.request.headers["Cookie"]
+
+ html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
+
+ cookies = HTTP::Cookies.from_client_headers(headers)
+ html.cookies.each do |cookie|
+ if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
+ if cookies[cookie.name]?
+ cookies[cookie.name] = cookie
+ else
+ cookies << cookie
+ end
+ end
+ end
+ headers = cookies.add_request_headers(headers)
+
+ if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
+ session_token = match["session_token"]
+ else
+ return env.redirect referer
+ end
+
+ headers["content-type"] = "application/x-www-form-urlencoded"
+ channel_req["session_token"] = session_token
+
+ subs = XML.parse_html(html.body)
+ subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel|
+ channel_id = channel.content.lstrip("/channel/").not_nil!
+ channel_req["channel_id"] = channel_id
+
+ YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
+ end
+ end
+
if redirect
env.redirect referer
else
diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr
index f2213da4..fe7e4e1c 100644
--- a/src/invidious/routes/playlists.cr
+++ b/src/invidious/routes/playlists.cr
@@ -163,20 +163,13 @@ module Invidious::Routes::Playlists
end
begin
- items = get_playlist_videos(playlist, offset: (page - 1) * 100)
+ videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
- items = [] of PlaylistVideo
+ videos = [] of PlaylistVideo
end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY)
- # Pagination
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/playlist?list=#{playlist.id}",
- current_page: page,
- show_next: (items.size == 100)
- )
-
templated "edit_playlist"
end
@@ -254,19 +247,11 @@ module Invidious::Routes::Playlists
begin
query = Invidious::Search::Query.new(env.params.query, :playlist, region)
- items = query.process.select(SearchVideo).map(&.as(SearchVideo))
+ videos = query.process.select(SearchVideo).map(&.as(SearchVideo))
rescue ex
- items = [] of SearchVideo
+ videos = [] of SearchVideo
end
- # Pagination
- query_encoded = URI.encode_www_form(query.try &.text || "", space_to_plus: true)
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/add_playlist_items?list=#{playlist.id}&q=#{query_encoded}",
- current_page: page,
- show_next: (items.size >= 20)
- )
-
env.set "add_playlist_items", plid
templated "add_playlist_items"
end
@@ -304,6 +289,23 @@ module Invidious::Routes::Playlists
end
end
+ if env.params.query["action_create_playlist"]?
+ action = "action_create_playlist"
+ elsif env.params.query["action_delete_playlist"]?
+ action = "action_delete_playlist"
+ elsif env.params.query["action_edit_playlist"]?
+ action = "action_edit_playlist"
+ elsif env.params.query["action_add_video"]?
+ action = "action_add_video"
+ video_id = env.params.query["video_id"]
+ elsif env.params.query["action_remove_video"]?
+ action = "action_remove_video"
+ elsif env.params.query["action_move_video_before"]?
+ action = "action_move_video_before"
+ else
+ return env.redirect referer
+ end
+
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
@@ -318,13 +320,21 @@ module Invidious::Routes::Playlists
end
end
- case action = env.params.query["action"]?
- when "add_video"
- if playlist.index.size >= CONFIG.playlist_length_limit
+ if !user.password
+ # TODO: Playlist stub, sync with YouTube for Google accounts
+ # playlist_ajax(playlist_id, action, env.request.headers)
+ end
+ email = user.email
+
+ case action
+ when "action_edit_playlist"
+ # TODO: Playlist stub
+ when "action_add_video"
+ if playlist.index.size >= 500
if redirect
- return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
+ return error_template(400, "Playlist cannot have more than 500 videos")
else
- return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
+ return error_json(400, "Playlist cannot have more than 500 videos")
end
end
@@ -356,14 +366,12 @@ module Invidious::Routes::Playlists
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
- when "remove_video"
+ when "action_remove_video"
index = env.params.query["set_video_id"]
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(playlist_id, index)
- when "move_video_before"
+ when "action_move_video_before"
# TODO: Playlist stub
- when nil
- return error_json(400, "Missing action")
else
return error_json(400, "Unsupported action #{action}")
end
@@ -402,13 +410,8 @@ module Invidious::Routes::Playlists
return error_template(500, ex)
end
- if playlist.is_a? InvidiousPlaylist
- page_count = (playlist.video_count / 100).to_i
- page_count += 1 if (playlist.video_count % 100) > 0
- else
- page_count = (playlist.video_count / 200).to_i
- page_count += 1 if (playlist.video_count % 200) > 0
- end
+ page_count = (playlist.video_count / 100).to_i
+ page_count += 1 if (playlist.video_count % 100) > 0
if page > page_count
return env.redirect "/playlist?list=#{plid}&page=#{page_count}"
@@ -419,11 +422,7 @@ module Invidious::Routes::Playlists
end
begin
- if playlist.is_a? InvidiousPlaylist
- items = get_playlist_videos(playlist, offset: (page - 1) * 100)
- else
- items = get_playlist_videos(playlist, offset: (page - 1) * 200)
- end
+ videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}")
end
@@ -432,13 +431,6 @@ module Invidious::Routes::Playlists
env.set "remove_playlist_items", plid
end
- # Pagination
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/playlist?list=#{playlist.id}",
- current_page: page,
- show_next: (page_count != 1 && page < page_count)
- )
-
templated "playlist"
end
diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr
index 39ca77c0..570cba69 100644
--- a/src/invidious/routes/preferences.cr
+++ b/src/invidious/routes/preferences.cr
@@ -27,10 +27,6 @@ module Invidious::Routes::PreferencesRoute
annotations_subscribed ||= "off"
annotations_subscribed = annotations_subscribed == "on"
- preload = env.params.body["preload"]?.try &.as(String)
- preload ||= "off"
- preload = preload == "on"
-
autoplay = env.params.body["autoplay"]?.try &.as(String)
autoplay ||= "off"
autoplay = autoplay == "on"
@@ -148,7 +144,6 @@ module Invidious::Routes::PreferencesRoute
preferences = Preferences.from_json({
annotations: annotations,
annotations_subscribed: annotations_subscribed,
- preload: preload,
autoplay: autoplay,
captions: captions,
comments: comments,
@@ -219,7 +214,7 @@ module Invidious::Routes::PreferencesRoute
statistics_enabled ||= "off"
CONFIG.statistics_enabled = statistics_enabled == "on"
- CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
+ CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
File.write("config/config.yml", CONFIG.to_yaml)
end
@@ -315,24 +310,6 @@ module Invidious::Routes::PreferencesRoute
response: error_template(415, "Invalid subscription file uploaded")
)
end
- when "import_youtube_pl"
- filename = part.filename || ""
- success = Invidious::User::Import.from_youtube_pl(user, body, filename, type)
-
- if !success
- haltf(env, status_code: 415,
- response: error_template(415, "Invalid playlist file uploaded")
- )
- end
- when "import_youtube_wh"
- filename = part.filename || ""
- success = Invidious::User::Import.from_youtube_wh(user, body, filename, type)
-
- if !success
- haltf(env, status_code: 415,
- response: error_template(415, "Invalid watch history file uploaded")
- )
- end
when "import_freetube"
Invidious::User::Import.from_freetube(user, body)
when "import_newpipe_subscriptions"
diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr
index b195c7b3..2a9705cf 100644
--- a/src/invidious/routes/search.cr
+++ b/src/invidious/routes/search.cr
@@ -51,39 +51,21 @@ module Invidious::Routes::Search
else
user = env.get? "user"
- # An URL was copy/pasted in the search box.
- # Redirect the user to the appropriate page.
- if query.url?
- return env.redirect UrlSanitizer.process(query.text).to_s
- end
-
begin
- if user
- items = query.process(user.as(User))
- else
- items = query.process
- end
+ videos = query.process
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex
return error_template(500, ex)
end
+ params = query.to_http_params
+ url_prev_page = "/search?#{params}&page=#{query.page - 1}"
+ url_next_page = "/search?#{params}&page=#{query.page + 1}"
+
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
- # Pagination
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/search?#{query.to_http_params}",
- current_page: query.page,
- show_next: (items.size >= 20)
- )
-
- if query.type == Invidious::Search::Query::Type::Channel
- env.set "search", "channel:#{query.channel} #{query.text}"
- else
- env.set "search", query.text
- end
-
+ env.set "search", query.text
templated "search"
end
end
@@ -105,18 +87,16 @@ module Invidious::Routes::Search
end
begin
- items = Invidious::Hashtag.fetch(hashtag, page)
+ videos = Invidious::Hashtag.fetch(hashtag, page)
rescue ex
return error_template(500, ex)
end
- # Pagination
+ params = env.params.query.empty? ? "" : "&#{env.params.query}"
+
hashtag_encoded = URI.encode_www_form(hashtag, space_to_plus: false)
- page_nav_html = Frontend::Pagination.nav_numeric(locale,
- base_url: "/hashtag/#{hashtag_encoded}",
- current_page: page,
- show_next: (items.size >= 60)
- )
+ url_prev_page = "/hashtag/#{hashtag_encoded}?page=#{page - 1}#{params}"
+ url_next_page = "/hashtag/#{hashtag_encoded}?page=#{page + 1}#{params}"
templated "hashtag"
end
diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr
index 1de655d2..7b1fa876 100644
--- a/src/invidious/routes/subscriptions.cr
+++ b/src/invidious/routes/subscriptions.cr
@@ -32,16 +32,29 @@ module Invidious::Routes::Subscriptions
end
end
+ if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1
+ action = "action_create_subscription_to_channel"
+ elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1
+ action = "action_remove_subscriptions"
+ else
+ return env.redirect referer
+ end
+
channel_id = env.params.query["c"]?
channel_id ||= ""
- case action = env.params.query["action"]?
- when "create_subscription_to_channel"
+ if !user.password
+ # Sync subscriptions with YouTube
+ subscribe_ajax(channel_id, action, env.request.headers)
+ end
+
+ case action
+ when "action_create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
- when "remove_subscriptions"
+ when "action_remove_subscriptions"
Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else
return error_json(400, "Unsupported action #{action}")
@@ -69,6 +82,14 @@ module Invidious::Routes::Subscriptions
user = user.as(User)
sid = sid.as(String)
+ if !user.password
+ # Refresh account
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env.request.headers["Cookie"]
+
+ user, sid = get_user(sid, headers)
+ end
+
action_takeout = env.params.query["action_takeout"]?.try &.to_i?
action_takeout ||= 0
action_takeout = action_takeout == 1
@@ -83,8 +104,33 @@ module Invidious::Routes::Subscriptions
if format == "json"
env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment"
+ playlists = Invidious::Database::Playlists.select_like_iv(user.email)
- return Invidious::User::Export.to_invidious(user)
+ return JSON.build do |json|
+ json.object do
+ json.field "subscriptions", user.subscriptions
+ json.field "watch_history", user.watched
+ json.field "preferences", user.preferences
+ json.field "playlists" do
+ json.array do
+ playlists.each do |playlist|
+ json.object do
+ json.field "title", playlist.title
+ json.field "description", html_to_content(playlist.description_html)
+ json.field "privacy", playlist.privacy.to_s
+ json.field "videos" do
+ json.array do
+ Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id|
+ json.string video_id
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
else
env.response.content_type = "application/xml"
env.response.headers["content-disposition"] = "attachment"
diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index 083087a9..560f9c19 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -21,7 +21,7 @@ module Invidious::Routes::VideoPlayback
end
# Sanity check, to avoid being used as an open proxy
- if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/)
+ if !host.matches?(/[\w-]+.googlevideo.com/)
return error_template(400, "Invalid \"host\" parameter.")
end
@@ -35,15 +35,7 @@ module Invidious::Routes::VideoPlayback
end
end
- # See: https://github.com/iv-org/invidious/issues/3302
- range_header = env.request.headers["Range"]?
- sq = query_params["sq"]?
- if range_header.nil? && sq.nil?
- range_for_head = query_params["range"]? || "0-640"
- headers["Range"] = "bytes=#{range_for_head}"
- end
-
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
@@ -58,7 +50,7 @@ module Invidious::Routes::VideoPlayback
if new_host != host
host = new_host
client.close
- client = make_client(URI.parse(new_host), region, force_resolve: true)
+ client = make_client(URI.parse(new_host), region)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
@@ -72,23 +64,15 @@ module Invidious::Routes::VideoPlayback
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region)
rescue ex
error = ex.message
end
end
- # Remove the Range header added previously.
- headers.delete("Range") if range_header.nil?
-
- playback_statistics = get_playback_statistic()
- playback_statistics["totalRequests"] += 1
-
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
- else
- playback_statistics["successfulRequests"] += 1
end
if url.includes? "&file=seg.ts"
@@ -107,8 +91,14 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
- return env.redirect url
+ location = URI.parse(location)
+ location = "#{location.request_target}&host=#{location.host}"
+
+ if region
+ location += "®ion=#{region}"
+ end
+
+ return env.redirect location
end
IO.copy(resp.body_io, env.response)
@@ -132,7 +122,7 @@ module Invidious::Routes::VideoPlayback
end
# TODO: Record bytes written so we can restart after a chunk fails
- loop do
+ while true
if !range_end && content_length
range_end = content_length
end
@@ -165,13 +155,10 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
+ location = URI.parse(location)
+ location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
- if title = query_params["title"]?
- url = "#{url}&title=#{URI.encode_www_form(title)}"
- end
-
- env.redirect url
+ env.redirect location
break
end
@@ -200,7 +187,7 @@ module Invidious::Routes::VideoPlayback
break
else
client.close
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region)
end
end
@@ -257,11 +244,6 @@ module Invidious::Routes::VideoPlayback
# YouTube /videoplayback links expire after 6 hours,
# so we have a mechanism here to redirect to the latest version
def self.latest_version(env)
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
- end
-
id = env.params.query["id"]?
itag = env.params.query["itag"]?.try &.to_i?
@@ -270,7 +252,7 @@ module Invidious::Routes::VideoPlayback
return error_template(400, "Invalid video ID")
end
- if !itag.nil? && (itag <= 0 || itag >= 1000)
+ if itag.nil? || itag <= 0 || itag >= 1000
return error_template(400, "Invalid itag")
end
@@ -291,11 +273,7 @@ module Invidious::Routes::VideoPlayback
return error_template(500, ex)
end
- if itag.nil?
- fmt = video.fmt_stream[-1]?
- else
- fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
- end
+ fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
url = fmt.try &.["url"]?.try &.as_s
if !url
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
index e777b3f1..fe1d8e54 100644
--- a/src/invidious/routes/watch.cr
+++ b/src/invidious/routes/watch.cr
@@ -30,6 +30,14 @@ module Invidious::Routes::Watch
return env.redirect "/"
end
+ embed_link = "/embed/#{id}"
+ if env.params.query.size > 1
+ embed_params = HTTP::Params.parse(env.params.query.to_s)
+ embed_params.delete_all("v")
+ embed_link += "?"
+ embed_link += embed_params.to_s
+ end
+
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(env.params.query, plid, id)
@@ -53,6 +61,8 @@ module Invidious::Routes::Watch
begin
video = get_video(id, region: params.region)
+ rescue ex : VideoRedirect
+ return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
LOGGER.error("get_video not found: #{id} : #{ex.message}")
return error_template(404, ex)
@@ -68,11 +78,11 @@ module Invidious::Routes::Watch
end
env.params.query.delete_all("iv_load_policy")
- if watched && preferences.watch_history
+ if watched && preferences.watch_history && !watched.includes? id
Invidious::Database::Users.mark_watched(user.as(User), id)
end
- if CONFIG.enable_user_notifications && notifications && notifications.includes? id
+ if notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
@@ -87,31 +97,31 @@ module Invidious::Routes::Watch
if source == "youtube"
begin
- comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
rescue ex
if preferences.comments[1] == "reddit"
- comments, reddit_thread = Comments.fetch_reddit(id)
- comment_html = Frontend::Comments.template_reddit(comments, locale)
+ comments, reddit_thread = fetch_reddit_comments(id)
+ comment_html = template_reddit_comments(comments, locale)
- comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
- comment_html = Comments.replace_links(comment_html)
+ comment_html = fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = replace_links(comment_html)
end
end
elsif source == "reddit"
begin
- comments, reddit_thread = Comments.fetch_reddit(id)
- comment_html = Frontend::Comments.template_reddit(comments, locale)
+ comments, reddit_thread = fetch_reddit_comments(id)
+ comment_html = template_reddit_comments(comments, locale)
- comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
- comment_html = Comments.replace_links(comment_html)
+ comment_html = fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = replace_links(comment_html)
rescue ex
if preferences.comments[1] == "youtube"
- comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
end
end
else
- comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
comment_html ||= ""
@@ -121,12 +131,10 @@ module Invidious::Routes::Watch
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
end
- # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -192,14 +200,6 @@ module Invidious::Routes::Watch
captions: video.captions
)
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- env.response.headers["Content-Security-Policy"] =
- env.response.headers["Content-Security-Policy"]
- .gsub("media-src", "media-src #{invidious_companion.public_url}")
- .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
- end
-
templated "watch"
end
@@ -251,10 +251,20 @@ module Invidious::Routes::Watch
end
end
- case action = env.params.query["action"]?
- when "mark_watched"
- Invidious::Database::Users.mark_watched(user, id)
- when "mark_unwatched"
+ if env.params.query["action_mark_watched"]?
+ action = "action_mark_watched"
+ elsif env.params.query["action_mark_unwatched"]?
+ action = "action_mark_unwatched"
+ else
+ return env.redirect referer
+ end
+
+ case action
+ when "action_mark_watched"
+ if !user.watched.includes? id
+ Invidious::Database::Users.mark_watched(user, id)
+ end
+ when "action_mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
else
return error_json(400, "Unsupported action #{action}")
@@ -277,12 +287,6 @@ module Invidious::Routes::Watch
return error_template(400, "Invalid clip ID") if response["error"]?
if video_id = response.dig?("endpoint", "watchEndpoint", "videoId")
- if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
- start_time, end_time, _ = parse_clip_parameters(params)
- env.params.query["start"] = start_time.to_s if start_time != nil
- env.params.query["end"] = end_time.to_s if end_time != nil
- end
-
return env.redirect "/watch?v=#{video_id}&#{env.params.query}"
else
return error_template(404, "The requested clip doesn't exist")
@@ -293,9 +297,6 @@ module Invidious::Routes::Watch
if CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.")
end
- if CONFIG.invidious_companion.present?
- return error_template(403, "Downloads should be routed through Companion when present")
- end
title = env.params.body["title"]? || ""
video_id = env.params.body["id"]? || ""
@@ -325,9 +326,10 @@ module Invidious::Routes::Watch
env.params.query["label"] = URI.decode_www_form(label.as_s)
return Invidious::Routes::API::V1::Videos.captions(env)
- elsif itag = download_widget["itag"]?.try &.as_i.to_s
+ elsif itag = download_widget["itag"]?.try &.as_i
# URL params specific to /latest_version
env.params.query["id"] = video_id
+ env.params.query["itag"] = itag.to_s
env.params.query["title"] = filename
env.params.query["local"] = "true"
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 46b71f1f..f409f13c 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -37,9 +37,7 @@ module Invidious::Routing
get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get
post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post
- if CONFIG.enable_user_notifications
- get "/modify_notifications", Routes::Notifications, :modify
- end
+ get "/modify_notifications", Routes::Notifications, :modify
{% end %}
self.register_image_routes
@@ -57,6 +55,7 @@ module Invidious::Routing
get "/login", Routes::Login, :login_page
post "/login", Routes::Login, :login
post "/signout", Routes::Login, :signout
+ get "/Captcha", Routes::Login, :captcha
# User preferences
get "/preferences", Routes::PreferencesRoute, :show
@@ -116,52 +115,29 @@ module Invidious::Routing
get "/channel/:ucid", Routes::Channels, :home
get "/channel/:ucid/home", Routes::Channels, :home
get "/channel/:ucid/videos", Routes::Channels, :videos
- get "/channel/:ucid/shorts", Routes::Channels, :shorts
- get "/channel/:ucid/streams", Routes::Channels, :streams
- get "/channel/:ucid/podcasts", Routes::Channels, :podcasts
- get "/channel/:ucid/releases", Routes::Channels, :releases
- get "/channel/:ucid/courses", Routes::Channels, :courses
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
- get "/channel/:ucid/posts", Routes::Channels, :community
- get "/channel/:ucid/channels", Routes::Channels, :channels
get "/channel/:ucid/about", Routes::Channels, :about
-
get "/channel/:ucid/live", Routes::Channels, :live
get "/user/:user/live", Routes::Channels, :live
get "/c/:user/live", Routes::Channels, :live
- get "/post/:id", Routes::Channels, :post
- # Channel catch-all, to redirect future routes to the channel's home
- # NOTE: defined last in order to be processed after the other routes
- get "/channel/:ucid/*", Routes::Channels, :redirect_home
-
- # /c/LinusTechTips
- get "/c/:user", Routes::Channels, :brand_redirect
- get "/c/:user/:tab", Routes::Channels, :brand_redirect
-
- # /user/linustechtips (Not always the same as /c/)
- get "/user/:user", Routes::Channels, :brand_redirect
- get "/user/:user/:tab", Routes::Channels, :brand_redirect
-
- # /@LinusTechTips (Handle)
- get "/@:user", Routes::Channels, :brand_redirect
- get "/@:user/:tab", Routes::Channels, :brand_redirect
-
- # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow
- get "/attribution_link", Routes::Channels, :brand_redirect
- get "/attribution_link/:tab", Routes::Channels, :brand_redirect
-
- # /profile?user=linustechtips
- get "/profile", Routes::Channels, :profile
- get "/profile/*", Routes::Channels, :profile
+ ["", "/videos", "/playlists", "/community", "/about"].each do |path|
+ # /c/LinusTechTips
+ get "/c/:user#{path}", Routes::Channels, :brand_redirect
+ # /user/linustechtips | Not always the same as /c/
+ get "/user/:user#{path}", Routes::Channels, :brand_redirect
+ # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow
+ get "/attribution_link#{path}", Routes::Channels, :brand_redirect
+ # /profile?user=linustechtips
+ get "/profile/#{path}", Routes::Channels, :profile
+ end
end
def register_watch_routes
get "/watch", Routes::Watch, :handle
post "/watch_ajax", Routes::Watch, :mark_watched
get "/watch/:id", Routes::Watch, :redirect
- get "/live/:id", Routes::Watch, :redirect
get "/shorts/:id", Routes::Watch, :redirect
get "/clip/:clip", Routes::Watch, :clip
get "/w/:id", Routes::Watch, :redirect
@@ -237,8 +213,6 @@ module Invidious::Routing
get "/api/v1/captions/:id", {{namespace}}::Videos, :captions
get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
- get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
- get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts
# Feeds
get "/api/v1/trending", {{namespace}}::Feeds, :trending
@@ -246,22 +220,10 @@ module Invidious::Routing
# Channels
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
- get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest
- get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos
- get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
- get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
- get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
- get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
- get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses
- get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
- get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
- get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community
- get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
- get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search
-
- # Posts
- get "/api/v1/post/:id", {{namespace}}::Channels, :post
- get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments
+ {% for route in {"videos", "latest", "playlists", "community", "search"} %}
+ get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
+ get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
+ {% end %}
# 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community
get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect
@@ -270,22 +232,17 @@ module Invidious::Routing
# Search
get "/api/v1/search", {{namespace}}::Search, :search
get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions
- get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag
-
# Authenticated
+ # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr
+ #
+ # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+
get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
- get "/api/v1/auth/export/invidious", {{namespace}}::Authenticated, :export_invidious
- post "/api/v1/auth/import/invidious", {{namespace}}::Authenticated, :import_invidious
-
- get "/api/v1/auth/history", {{namespace}}::Authenticated, :get_history
- post "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_watched
- delete "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_unwatched
- delete "/api/v1/auth/history", {{namespace}}::Authenticated, :clear_history
-
get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed
get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions
@@ -303,17 +260,14 @@ module Invidious::Routing
post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token
post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token
- if CONFIG.enable_user_notifications
- get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
- post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
- end
+ get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
# Misc
get "/api/v1/stats", {{namespace}}::Misc, :stats
get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist
get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist
get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes
- get "/api/v1/resolveurl", {{namespace}}::Misc, :resolve_url
{% end %}
end
end
diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr
index bc2715cf..c2b5c758 100644
--- a/src/invidious/search/filters.cr
+++ b/src/invidious/search/filters.cr
@@ -75,7 +75,7 @@ module Invidious::Search
@type : Type = Type::All,
@duration : Duration = Duration::None,
@features : Features = Features::None,
- @sort : Sort = Sort::Relevance,
+ @sort : Sort = Sort::Relevance
)
end
@@ -300,9 +300,9 @@ module Invidious::Search
object["9:varint"] = ((page - 1) * 20).to_i64
end
- # Prevent censoring of self harm topics
- # See https://github.com/iv-org/invidious/issues/4398
- object["30:varint"] = 1.to_i64
+ # If the object is empty, return an empty string,
+ # otherwise encode to protobuf then to base64
+ return "" if object.empty?
return object
.try { |i| Protodec::Any.cast_json(i) }
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index 25edb936..d1409c06 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -9,8 +9,7 @@ module Invidious::Search
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
- items, _ = extract_items(initial_data)
- return items.reject!(Category)
+ return extract_items(initial_data)
end
# Search a youtube channel
@@ -31,8 +30,17 @@ module Invidious::Search
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
- items, _ = extract_items(response_json, "", ucid)
- return items.reject!(Category)
+ continuation_items = response_json["onResponseReceivedActions"]?
+ .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
+
+ return [] of SearchItem if !continuation_items
+
+ items = [] of SearchItem
+ continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
+ extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
+ end
+
+ return items
end
# Search inside of user subscriptions
diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr
index 94a92e23..24e79609 100644
--- a/src/invidious/search/query.cr
+++ b/src/invidious/search/query.cr
@@ -20,9 +20,6 @@ module Invidious::Search
property region : String?
property channel : String = ""
- # Flag that indicates if the smart search features have been disabled.
- @inhibit_ssf : Bool = false
-
# Return true if @raw_query is either `nil` or empty
private def empty_raw_query?
return @raw_query.empty?
@@ -47,22 +44,14 @@ module Invidious::Search
def initialize(
params : HTTP::Params,
@type : Type = Type::Regular,
- @region : String? = nil,
+ @region : String? = nil
)
# Get the raw search query string (common to all search types). In
# Regular search mode, also look for the `search_query` URL parameter
- _raw_query = params["q"]?
- _raw_query ||= params["search_query"]? if @type.regular?
- _raw_query ||= ""
-
- # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs.
- @raw_query = _raw_query.strip
-
- # Check for smart features (ex: URL search) inhibitor (backslash).
- # If inhibitor is present, remove it.
- if @raw_query.starts_with?('\\')
- @inhibit_ssf = true
- @raw_query = @raw_query[1..]
+ if @type.regular?
+ @raw_query = params["q"]? || params["search_query"]? || ""
+ else
+ @raw_query = params["q"]? || ""
end
# Get the page number (also common to all search types)
@@ -96,7 +85,7 @@ module Invidious::Search
@filters = Filters.from_iv_params(params)
@channel = params["channel"]? || ""
- if @filters.default? && @raw_query.index(/\w:\w/)
+ if @filters.default? && @raw_query.includes?(':')
# Parse legacy filters from query
@filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query)
else
@@ -124,7 +113,7 @@ module Invidious::Search
case @type
when .regular?, .playlist?
- items = Processors.regular(self)
+ items = unnest_items(Processors.regular(self))
#
when .channel?
items = Processors.channel(self)
@@ -148,21 +137,25 @@ module Invidious::Search
return params
end
- # Checks if the query is a standalone URL
- def url? : Bool
- # If the smart features have been inhibited, don't go further.
- return false if @inhibit_ssf
+ # TODO: clean code
+ private def unnest_items(all_items) : Array(SearchItem)
+ items = [] of SearchItem
- # Only supported in regular search mode
- return false if !@type.regular?
+ # Light processing to flatten search results out of Categories.
+ # They should ideally be supported in the future.
+ all_items.each do |i|
+ if i.is_a? Category
+ i.contents.each do |nest_i|
+ if !nest_i.is_a? Video
+ items << nest_i
+ end
+ end
+ else
+ items << i
+ end
+ end
- # If filters are present, that's a regular search
- return false if !@filters.default?
-
- # Simple heuristics: domain name
- return @raw_query.starts_with?(
- /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\//
- )
+ return items
end
end
end
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index d14cde5d..1f957081 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -4,12 +4,11 @@ def fetch_trending(trending_type, region, locale)
plid = nil
- case trending_type.try &.downcase
- when "music"
+ if trending_type == "Music"
params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D"
- when "gaming"
+ elsif trending_type == "Gaming"
params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D"
- when "movies"
+ elsif trending_type == "Movies"
params = "4gIKGgh0cmFpbGVycw%3D%3D"
else # Default
params = ""
@@ -17,26 +16,7 @@ def fetch_trending(trending_type, region, locale)
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.browse("FEtrending", params: params, client_config: client_config)
+ trending = extract_videos(initial_data)
- items, _ = extract_items(initial_data)
-
- extracted = [] of SearchItem
-
- deduplicate = items.size > 1
-
- items.each do |itm|
- if itm.is_a?(Category)
- # Ignore the smaller categories, as they generally contain a sponsored
- # channel, which brings a lot of noise on the trending page.
- # See: https://github.com/iv-org/invidious/issues/2989
- next if (itm.contents.size < 24 && deduplicate)
-
- extracted.concat itm.contents.select(SearchItem)
- else
- extracted << itm
- end
- end
-
- # Deduplicate items before returning results
- return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
+ return {trending, plid}
end
diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr
index b175c3b9..8a0f67e5 100644
--- a/src/invidious/user/captcha.cr
+++ b/src/invidious/user/captcha.cr
@@ -4,6 +4,8 @@ struct Invidious::User
module Captcha
extend self
+ private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com")
+
def generate_image(key)
second = Random::Secure.rand(12)
second_angle = second * 30
@@ -58,5 +60,19 @@ struct Invidious::User
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)},
}
end
+
+ def generate_text(key)
+ response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body)
+ response = JSON.parse(response)
+
+ tokens = response["a"].as_a.map do |answer|
+ generate_response(answer.as_s, {":login"}, key, use_nonce: true)
+ end
+
+ return {
+ question: response["q"].as_s,
+ tokens: tokens,
+ }
+ end
end
end
diff --git a/src/invidious/user/exports.cr b/src/invidious/user/exports.cr
deleted file mode 100644
index b52503c9..00000000
--- a/src/invidious/user/exports.cr
+++ /dev/null
@@ -1,35 +0,0 @@
-struct Invidious::User
- module Export
- extend self
-
- def to_invidious(user : User)
- playlists = Invidious::Database::Playlists.select_like_iv(user.email)
-
- return JSON.build do |json|
- json.object do
- json.field "subscriptions", user.subscriptions
- json.field "watch_history", user.watched
- json.field "preferences", user.preferences
- json.field "playlists" do
- json.array do
- playlists.each do |playlist|
- json.object do
- json.field "title", playlist.title
- json.field "description", html_to_content(playlist.description_html)
- json.field "privacy", playlist.privacy.to_s
- json.field "videos" do
- json.array do
- Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: CONFIG.playlist_length_limit).each do |video_id|
- json.string video_id
- end
- end
- end
- end
- end
- end
- end
- end
- end
- end
- end # module
-end
diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr
index 007eb666..f8b9e4e4 100644
--- a/src/invidious/user/imports.cr
+++ b/src/invidious/user/imports.cr
@@ -6,7 +6,7 @@ struct Invidious::User
# Parse a youtube CSV subscription file
def parse_subscription_export_csv(csv_content : String)
- rows = CSV.new(csv_content.strip('\n'), headers: true)
+ rows = CSV.new(csv_content, headers: true)
subscriptions = Array(String).new
# Counter to limit the amount of imports.
@@ -30,60 +30,6 @@ struct Invidious::User
return subscriptions
end
- def parse_playlist_export_csv(user : User, raw_input : String)
- # Split the input into head and body content
- raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true)
-
- # Create the playlist from the head content
- csv_head = CSV.new(raw_head.strip('\n'), headers: true)
- csv_head.next
- title = csv_head[4]
- description = csv_head[5]
- visibility = csv_head[6]
-
- if visibility.compare("Public", case_insensitive: true) == 0
- privacy = PlaylistPrivacy::Public
- else
- privacy = PlaylistPrivacy::Private
- end
-
- playlist = create_playlist(title, privacy, user)
- Invidious::Database::Playlists.update_description(playlist.id, description)
-
- # Add each video to the playlist from the body content
- csv_body = CSV.new(raw_body.strip('\n'), headers: true)
- csv_body.each do |row|
- video_id = row[0]
- if playlist
- next if !video_id
- next if video_id == "Video Id"
-
- begin
- video = get_video(video_id)
- rescue ex
- next
- end
-
- playlist_video = PlaylistVideo.new({
- title: video.title,
- id: video.id,
- author: video.author,
- ucid: video.ucid,
- length_seconds: video.length_seconds,
- published: video.published,
- plid: playlist.id,
- live_now: video.live_now,
- index: Random::Secure.rand(0_i64..Int64::MAX),
- })
-
- Invidious::Database::PlaylistVideos.insert(playlist_video)
- Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
- end
- end
-
- return playlist
- end
-
# -------------------
# Invidious
# -------------------
@@ -102,7 +48,7 @@ struct Invidious::User
if data["watch_history"]?
user.watched += data["watch_history"].as_a.map(&.as_s)
- user.watched.reverse!.uniq!.reverse!
+ user.watched.uniq!
Invidious::Database::Users.update_watch_history(user)
end
@@ -115,7 +61,7 @@ struct Invidious::User
playlists.each do |item|
title = item["title"]?.try &.as_s?.try &.delete("<>")
description = item["description"]?.try &.as_s?.try &.delete("\r")
- privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state }
+ privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
next if !title
next if !description
@@ -124,16 +70,14 @@ struct Invidious::User
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
- item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
- if idx > CONFIG.playlist_length_limit
- raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
- end
+ videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
+ raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
video_id = video_id.try &.as_s?
next if !video_id
begin
- video = get_video(video_id, false)
+ video = get_video(video_id)
rescue ex
next
end
@@ -161,7 +105,7 @@ struct Invidious::User
# Youtube
# -------------------
- private def opml?(mimetype : String, extension : String)
+ private def is_opml?(mimetype : String, extension : String)
opml_mimetypes = [
"application/xml",
"text/xml",
@@ -179,10 +123,10 @@ struct Invidious::User
def from_youtube(user : User, body : String, filename : String, type : String) : Bool
extension = filename.split(".").last
- if opml?(type, extension)
+ if is_opml?(type, extension)
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
- channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0]
+ channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body)
@@ -203,41 +147,6 @@ struct Invidious::User
return true
end
- def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool
- extension = filename.split(".").last
-
- if extension == "csv" || type == "text/csv"
- playlist = parse_playlist_export_csv(user, body)
- if playlist
- return true
- else
- return false
- end
- else
- return false
- end
- end
-
- def from_youtube_wh(user : User, body : String, filename : String, type : String) : Bool
- extension = filename.split(".").last
-
- if extension == "json" || type == "application/json"
- data = JSON.parse(body)
- watched = data.as_a.compact_map do |item|
- next unless url = item["titleUrl"]?
- next unless match = url.as_s.match(/\?v=(?[a-zA-Z0-9_-]+)$/)
- match["video_id"]
- end
- watched.reverse! # YouTube have newest first
- user.watched += watched
- user.watched.uniq!
- Invidious::Database::Users.update_watch_history(user)
- return true
- else
- return false
- end
- end
-
# -------------------
# Freetube
# -------------------
@@ -248,12 +157,8 @@ struct Invidious::User
subs = matches.map(&.["channel_id"])
if subs.empty?
- profiles = body.split('\n', remove_empty: true)
- profiles.each do |profile|
- if data = JSON.parse(profile)["subscriptions"]?
- subs += data.as_a.map(&.["id"].as_s)
- end
- end
+ data = JSON.parse(body)["subscriptions"]
+ subs = data.as_a.map(&.["id"].as_s)
end
user.subscriptions += subs
@@ -290,39 +195,42 @@ struct Invidious::User
end
def from_newpipe(user : User, body : String) : Bool
- Compress::Zip::File.open(IO::Memory.new(body), true) do |file|
- entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" }
- return false if entry.nil?
- entry.open do |file_io|
- # Ensure max size of 4MB
- io_sized = IO::Sized.new(file_io, 0x400000)
+ io = IO::Memory.new(body)
- begin
- temp = File.tempfile(".db") do |tempfile|
- begin
- File.write(tempfile.path, io_sized.gets_to_end)
- rescue
- return false
- end
+ Compress::Zip::File.open(io) do |file|
+ file.entries.each do |entry|
+ entry.open do |file_io|
+ # Ensure max size of 4MB
+ io_sized = IO::Sized.new(file_io, 0x400000)
- DB.open("sqlite3://" + tempfile.path) do |db|
- user.watched += db.query_all("SELECT url FROM streams", as: String)
- .map(&.lchop("https://www.youtube.com/watch?v="))
+ next if entry.filename != "newpipe.db"
- user.watched.uniq!
- Invidious::Database::Users.update_watch_history(user)
+ tempfile = File.tempfile(".db")
- user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
- .map(&.lchop("https://www.youtube.com/channel/"))
-
- user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions)
-
- Invidious::Database::Users.update_subscriptions(user)
- end
+ begin
+ File.write(tempfile.path, io_sized.gets_to_end)
+ rescue
+ return false
end
- ensure
- temp.delete if !temp.nil?
+
+ db = DB.open("sqlite3://" + tempfile.path)
+
+ user.watched += db.query_all("SELECT url FROM streams", as: String)
+ .map(&.lchop("https://www.youtube.com/watch?v="))
+
+ user.watched.uniq!
+ Invidious::Database::Users.update_watch_history(user)
+
+ user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
+ .map(&.lchop("https://www.youtube.com/channel/"))
+
+ user.subscriptions.uniq!
+ user.subscriptions = get_batch_channels(user.subscriptions)
+
+ Invidious::Database::Users.update_subscriptions(user)
+
+ db.close
+ tempfile.delete
end
end
end
diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr
index 0a8525f3..b3059403 100644
--- a/src/invidious/user/preferences.cr
+++ b/src/invidious/user/preferences.cr
@@ -4,7 +4,6 @@ struct Preferences
property annotations : Bool = CONFIG.default_user_preferences.annotations
property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
- property preload : Bool = CONFIG.default_user_preferences.preload
property autoplay : Bool = CONFIG.default_user_preferences.autoplay
property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index 65566d20..b763596b 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -3,6 +3,75 @@ require "crypto/bcrypt/password"
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
+def get_user(sid, headers, refresh = true)
+ if email = Invidious::Database::SessionIDs.select_email(sid)
+ user = Invidious::Database::Users.select!(email: email)
+
+ if refresh && Time.utc - user.updated > 1.minute
+ user, sid = fetch_user(sid, headers)
+
+ Invidious::Database::Users.insert(user, update_on_conflict: true)
+ Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
+
+ begin
+ view_name = "subscriptions_#{sha256(user.email)}"
+ PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
+ rescue ex
+ end
+ end
+ else
+ user, sid = fetch_user(sid, headers)
+
+ Invidious::Database::Users.insert(user, update_on_conflict: true)
+ Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
+
+ begin
+ view_name = "subscriptions_#{sha256(user.email)}"
+ PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
+ rescue ex
+ end
+ end
+
+ return user, sid
+end
+
+def fetch_user(sid, headers)
+ feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
+ feed = XML.parse_html(feed.body)
+
+ channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
+ if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
+ nil
+ else
+ channel["href"].lstrip("/channel/")
+ end
+ end
+
+ channels = get_batch_channels(channels)
+
+ email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
+ if email
+ email = email.content.strip
+ else
+ email = ""
+ end
+
+ token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
+
+ user = Invidious::User.new({
+ updated: Time.utc,
+ notifications: [] of String,
+ subscriptions: channels,
+ email: email,
+ preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
+ password: nil,
+ token: token,
+ watched: [] of String,
+ feed_needs_update: true,
+ })
+ return user, sid
+end
+
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
@@ -22,6 +91,38 @@ def create_user(sid, email, password)
return user, sid
end
+def subscribe_ajax(channel_id, action, env_headers)
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env_headers["Cookie"]
+
+ html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
+
+ cookies = HTTP::Cookies.from_client_headers(headers)
+ html.cookies.each do |cookie|
+ if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
+ if cookies[cookie.name]?
+ cookies[cookie.name] = cookie
+ else
+ cookies << cookie
+ end
+ end
+ end
+ headers = cookies.add_request_headers(headers)
+
+ if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
+ session_token = match["session_token"]
+
+ headers["content-type"] = "application/x-www-form-urlencoded"
+
+ post_req = {
+ session_token: session_token,
+ }
+ post_url = "/subscription_ajax?#{action}=1&c=#{channel_id}"
+
+ YT_POOL.client &.post(post_url, headers, form: post_req)
+ end
+end
+
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 348a0a66..c0ed6e85 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -1,22 +1,280 @@
-enum VideoType
- Video
- Livestream
- Scheduled
+CAPTION_LANGUAGES = {
+ "",
+ "English",
+ "English (auto-generated)",
+ "English (United Kingdom)",
+ "English (United States)",
+ "Afrikaans",
+ "Albanian",
+ "Amharic",
+ "Arabic",
+ "Armenian",
+ "Azerbaijani",
+ "Bangla",
+ "Basque",
+ "Belarusian",
+ "Bosnian",
+ "Bulgarian",
+ "Burmese",
+ "Cantonese (Hong Kong)",
+ "Catalan",
+ "Cebuano",
+ "Chinese",
+ "Chinese (China)",
+ "Chinese (Hong Kong)",
+ "Chinese (Simplified)",
+ "Chinese (Taiwan)",
+ "Chinese (Traditional)",
+ "Corsican",
+ "Croatian",
+ "Czech",
+ "Danish",
+ "Dutch",
+ "Dutch (auto-generated)",
+ "Esperanto",
+ "Estonian",
+ "Filipino",
+ "Finnish",
+ "French",
+ "French (auto-generated)",
+ "Galician",
+ "Georgian",
+ "German",
+ "German (auto-generated)",
+ "Greek",
+ "Gujarati",
+ "Haitian Creole",
+ "Hausa",
+ "Hawaiian",
+ "Hebrew",
+ "Hindi",
+ "Hmong",
+ "Hungarian",
+ "Icelandic",
+ "Igbo",
+ "Indonesian",
+ "Indonesian (auto-generated)",
+ "Interlingue",
+ "Irish",
+ "Italian",
+ "Italian (auto-generated)",
+ "Japanese",
+ "Japanese (auto-generated)",
+ "Javanese",
+ "Kannada",
+ "Kazakh",
+ "Khmer",
+ "Korean",
+ "Korean (auto-generated)",
+ "Kurdish",
+ "Kyrgyz",
+ "Lao",
+ "Latin",
+ "Latvian",
+ "Lithuanian",
+ "Luxembourgish",
+ "Macedonian",
+ "Malagasy",
+ "Malay",
+ "Malayalam",
+ "Maltese",
+ "Maori",
+ "Marathi",
+ "Mongolian",
+ "Nepali",
+ "Norwegian Bokmål",
+ "Nyanja",
+ "Pashto",
+ "Persian",
+ "Polish",
+ "Portuguese",
+ "Portuguese (auto-generated)",
+ "Portuguese (Brazil)",
+ "Punjabi",
+ "Romanian",
+ "Russian",
+ "Russian (auto-generated)",
+ "Samoan",
+ "Scottish Gaelic",
+ "Serbian",
+ "Shona",
+ "Sindhi",
+ "Sinhala",
+ "Slovak",
+ "Slovenian",
+ "Somali",
+ "Southern Sotho",
+ "Spanish",
+ "Spanish (auto-generated)",
+ "Spanish (Latin America)",
+ "Spanish (Mexico)",
+ "Spanish (Spain)",
+ "Sundanese",
+ "Swahili",
+ "Swedish",
+ "Tajik",
+ "Tamil",
+ "Telugu",
+ "Thai",
+ "Turkish",
+ "Turkish (auto-generated)",
+ "Ukrainian",
+ "Urdu",
+ "Uzbek",
+ "Vietnamese",
+ "Vietnamese (auto-generated)",
+ "Welsh",
+ "Western Frisian",
+ "Xhosa",
+ "Yiddish",
+ "Yoruba",
+ "Zulu",
+}
+
+REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"}
+
+# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
+VIDEO_FORMATS = {
+ "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
+ "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
+ "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
+ "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
+ "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
+ "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+
+ "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
+ "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+
+ # 3D videos
+ "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+
+ # Apple HTTP Live Streaming
+ "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
+ "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
+ "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
+
+ # DASH mp4 video
+ "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
+ "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
+ "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
+ "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
+ "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
+ "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
+ "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
+ "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
+ "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
+ "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
+ "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
+ "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
+
+ # Dash mp4 audio
+ "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
+ "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
+ "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
+ "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
+ "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
+ "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
+ "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
+
+ # Dash webm
+ "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
+ "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
+ "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
+ "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
+ "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
+ "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
+ # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
+ "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
+ "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
+ "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+
+ # Dash webm audio
+ "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
+ "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
+
+ # Dash webm audio with opus inside
+ "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
+ "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
+ "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
+
+ # av01 video only formats sometimes served with "unknown" codecs
+ "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
+ "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
+ "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
+ "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
+}
+
+struct VideoPreferences
+ include JSON::Serializable
+
+ property annotations : Bool
+ property autoplay : Bool
+ property comments : Array(String)
+ property continue : Bool
+ property continue_autoplay : Bool
+ property controls : Bool
+ property listen : Bool
+ property local : Bool
+ property preferred_captions : Array(String)
+ property player_style : String
+ property quality : String
+ property quality_dash : String
+ property raw : Bool
+ property region : String?
+ property related_videos : Bool
+ property speed : Float32 | Float64
+ property video_end : Float64 | Int32
+ property video_loop : Bool
+ property extend_desc : Bool
+ property video_start : Float64 | Int32
+ property volume : Int32
+ property vr_mode : Bool
+ property save_player_pos : Bool
end
struct Video
include DB::Serializable
- # Version of the JSON structure
- # It prevents us from loading an incompatible version from cache
- # (either newer or older, if instances with different versions run
- # concurrently, e.g during a version upgrade rollout).
- #
- # NOTE: don't forget to bump this number if any change is made to
- # the `params` structure in videos/parser.cr!!!
- #
- SCHEMA_VERSION = 3
-
property id : String
@[DB::Field(converter: Video::JSONConverter)]
@@ -24,7 +282,13 @@ struct Video
property updated : Time
@[DB::Field(ignore: true)]
- @captions = [] of Invidious::Videos::Captions::Metadata
+ property captions : Array(Caption)?
+
+ @[DB::Field(ignore: true)]
+ property adaptive_fmts : Array(Hash(String, JSON::Any))?
+
+ @[DB::Field(ignore: true)]
+ property fmt_stream : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property description : String?
@@ -35,49 +299,289 @@ struct Video
end
end
- # Methods for API v1 JSON
-
def to_json(locale : String?, json : JSON::Builder)
- Invidious::JSONify::APIv1.video(self, json, locale: locale)
+ json.object do
+ json.field "type", "video"
+
+ json.field "title", self.title
+ json.field "videoId", self.id
+
+ json.field "error", info["reason"] if info["reason"]?
+
+ json.field "videoThumbnails" do
+ generate_thumbnails(json, self.id)
+ end
+ json.field "storyboards" do
+ generate_storyboards(json, self.id, self.storyboards)
+ end
+
+ json.field "description", self.description
+ json.field "descriptionHtml", self.description_html
+ json.field "published", self.published.to_unix
+ json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
+ json.field "keywords", self.keywords
+
+ json.field "viewCount", self.views
+ json.field "likeCount", self.likes
+ json.field "dislikeCount", 0_i64
+
+ json.field "paid", self.paid
+ json.field "premium", self.premium
+ json.field "isFamilyFriendly", self.is_family_friendly
+ json.field "allowedRegions", self.allowed_regions
+ json.field "genre", self.genre
+ json.field "genreUrl", self.genre_url
+
+ json.field "author", self.author
+ json.field "authorId", self.ucid
+ json.field "authorUrl", "/channel/#{self.ucid}"
+
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+
+ json.field "subCountText", self.sub_count_text
+
+ json.field "lengthSeconds", self.length_seconds
+ json.field "allowRatings", self.allow_ratings
+ json.field "rating", 0_i64
+ json.field "isListed", self.is_listed
+ json.field "liveNow", self.live_now
+ json.field "isUpcoming", self.is_upcoming
+
+ if self.premiere_timestamp
+ json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
+ end
+
+ if hlsvp = self.hls_manifest_url
+ hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
+ json.field "hlsUrl", hlsvp
+ end
+
+ json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
+
+ json.field "adaptiveFormats" do
+ json.array do
+ self.adaptive_fmts.each do |fmt|
+ json.object do
+ # Only available on regular videos, not livestreams/OTF streams
+ if init_range = fmt["initRange"]?
+ json.field "init", "#{init_range["start"]}-#{init_range["end"]}"
+ end
+ if index_range = fmt["indexRange"]?
+ json.field "index", "#{index_range["start"]}-#{index_range["end"]}"
+ end
+
+ # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only)
+ json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
+
+ json.field "url", fmt["url"]
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
+ json.field "clen", fmt["contentLength"]? || "-1"
+ json.field "lmt", fmt["lastModified"]
+ json.field "projectionType", fmt["projectionType"]
+
+ if fmt_info = itag_to_metadata?(fmt["itag"])
+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+ json.field "fps", fps
+ json.field "container", fmt_info["ext"]
+ json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
+
+ if fmt_info["height"]?
+ json.field "resolution", "#{fmt_info["height"]}p"
+
+ quality_label = "#{fmt_info["height"]}p"
+ if fps > 30
+ quality_label += "60"
+ end
+ json.field "qualityLabel", quality_label
+
+ if fmt_info["width"]?
+ json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
+ end
+ end
+ end
+
+ # Livestream chunk infos
+ json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec")
+ json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec")
+
+ # Audio-related data
+ json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality")
+ json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate")
+ json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels")
+
+ # Extra misc stuff
+ json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo")
+ json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack")
+ end
+ end
+ end
+ end
+
+ json.field "formatStreams" do
+ json.array do
+ self.fmt_stream.each do |fmt|
+ json.object do
+ json.field "url", fmt["url"]
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
+ json.field "quality", fmt["quality"]
+
+ fmt_info = itag_to_metadata?(fmt["itag"])
+ if fmt_info
+ fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+ json.field "fps", fps
+ json.field "container", fmt_info["ext"]
+ json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
+
+ if fmt_info["height"]?
+ json.field "resolution", "#{fmt_info["height"]}p"
+
+ quality_label = "#{fmt_info["height"]}p"
+ if fps > 30
+ quality_label += "60"
+ end
+ json.field "qualityLabel", quality_label
+
+ if fmt_info["width"]?
+ json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ json.field "captions" do
+ json.array do
+ self.captions.each do |caption|
+ json.object do
+ json.field "label", caption.name
+ json.field "language_code", caption.language_code
+ json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}"
+ end
+ end
+ end
+ end
+
+ json.field "recommendedVideos" do
+ json.array do
+ self.related_videos.each do |rv|
+ if rv["id"]?
+ json.object do
+ json.field "videoId", rv["id"]
+ json.field "title", rv["title"]
+ json.field "videoThumbnails" do
+ generate_thumbnails(json, rv["id"])
+ end
+
+ json.field "author", rv["author"]
+ json.field "authorUrl", "/channel/#{rv["ucid"]?}"
+ json.field "authorId", rv["ucid"]?
+ if rv["author_thumbnail"]?
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+ end
+
+ json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
+ json.field "viewCountText", rv["short_view_count"]?
+ json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
+ end
+ end
+ end
+ end
+ end
+ end
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
- JSON.build do |json|
- Invidious::JSONify::APIv1.video(self, json, locale: locale)
- end
+ JSON.build { |json| to_json(locale, json) }
end
def to_json(json : JSON::Builder | Nil = nil)
to_json(nil, json)
end
- # Misc methods
-
- def video_type : VideoType
- video_type = info["videoType"]?.try &.as_s || "video"
- return VideoType.parse?(video_type) || VideoType::Video
+ def title
+ info["videoDetails"]["title"]?.try &.as_s || ""
end
- def schema_version : Int
- return info["version"]?.try &.as_i || 1
+ def ucid
+ info["videoDetails"]["channelId"]?.try &.as_s || ""
+ end
+
+ def author
+ info["videoDetails"]["author"]?.try &.as_s || ""
+ end
+
+ def length_seconds : Int32
+ info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i ||
+ info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
+ end
+
+ def views : Int64
+ info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
+ end
+
+ def likes : Int64
+ info["likes"]?.try &.as_i64 || 0_i64
+ end
+
+ def dislikes : Int64
+ info["dislikes"]?.try &.as_i64 || 0_i64
end
def published : Time
- return info["published"]?
+ info
+ .dig?("microformat", "playerMicroformatRenderer", "publishDate")
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
end
def published=(other : Time)
- info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
+ info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
+ end
+
+ def allow_ratings
+ r = info["videoDetails"]["allowRatings"]?.try &.as_bool
+ r.nil? ? false : r
end
def live_now
- return (self.video_type == VideoType::Livestream)
+ info["microformat"]?.try &.["playerMicroformatRenderer"]?
+ .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false
end
- def post_live_dvr
- return info["isPostLiveDvr"].as_bool
+ def is_listed
+ info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
+ end
+
+ def is_upcoming
+ info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
end
def premiere_timestamp : Time?
@@ -86,30 +590,71 @@ struct Video
.try { |t| Time.parse_rfc3339(t.as_s) }
end
+ def keywords
+ info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
+ end
+
def related_videos
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
end
- # Methods for parsing streaming data
-
- def fmt_stream : Array(Hash(String, JSON::Any))
- if formats = info.dig?("streamingData", "formats")
- return formats
- .as_a.map(&.as_h)
- .sort_by! { |f| f["width"]?.try &.as_i || 0 }
- else
- return [] of Hash(String, JSON::Any)
- end
+ def allowed_regions
+ info
+ .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
+ .try &.as_a.map &.as_s || [] of String
end
- def adaptive_fmts : Array(Hash(String, JSON::Any))
- if formats = info.dig?("streamingData", "adaptiveFormats")
- return formats
- .as_a.map(&.as_h)
- .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 }
- else
- return [] of Hash(String, JSON::Any)
+ def author_thumbnail : String
+ info["authorThumbnail"]?.try &.as_s || ""
+ end
+
+ def author_verified : Bool
+ info["authorVerified"]?.try &.as_bool || false
+ end
+
+ def sub_count_text : String
+ info["subCountText"]?.try &.as_s || "-"
+ end
+
+ def fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
+
+ fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
end
+
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @fmt_stream = fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any)))
+ end
+
+ def adaptive_fmts
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
+ fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
+ end
+
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @adaptive_fmts = fmt_stream
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
@@ -120,57 +665,143 @@ struct Video
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
- # Misc. methods
-
def storyboards
- container = info.dig?("storyboards") || JSON::Any.new("{}")
- return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds)
+ storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
+ .try &.as_s.split("|")
+
+ if !storyboards
+ if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
+ return [{
+ url: storyboard.split("#")[0],
+ width: 106,
+ height: 60,
+ count: -1,
+ interval: 5000,
+ storyboard_width: 3,
+ storyboard_height: 3,
+ storyboard_count: -1,
+ }]
+ end
+ end
+
+ items = [] of NamedTuple(
+ url: String,
+ width: Int32,
+ height: Int32,
+ count: Int32,
+ interval: Int32,
+ storyboard_width: Int32,
+ storyboard_height: Int32,
+ storyboard_count: Int32)
+
+ return items if !storyboards
+
+ url = URI.parse(storyboards.shift)
+ params = HTTP::Params.parse(url.query || "")
+
+ storyboards.each_with_index do |sb, i|
+ width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
+ params["sigh"] = sigh
+ url.query = params.to_s
+
+ width = width.to_i
+ height = height.to_i
+ count = count.to_i
+ interval = interval.to_i
+ storyboard_width = storyboard_width.to_i
+ storyboard_height = storyboard_height.to_i
+ storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
+
+ items << {
+ url: url.to_s.sub("$L", i).sub("$N", "M$M"),
+ width: width,
+ height: height,
+ count: count,
+ interval: interval,
+ storyboard_width: storyboard_width,
+ storyboard_height: storyboard_height,
+ storyboard_count: storyboard_count,
+ }
+ end
+
+ items
end
def paid
- return (self.reason || "").includes? "requires payment"
+ reason = info.dig?("playabilityStatus", "reason").try &.as_s || ""
+ return reason.includes? "requires payment"
end
def premium
keywords.includes? "YouTube Red"
end
- def captions : Array(Invidious::Videos::Captions::Metadata)
- if @captions.empty? && @info.has_key?("captions")
- @captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"])
- end
+ def captions : Array(Caption)
+ return @captions.as(Array(Caption)) if @captions
+ captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
+ name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
+ language_code = caption["languageCode"].to_s
+ base_url = caption["baseUrl"].to_s
- return @captions
+ caption = Caption.new(name.to_s, language_code, base_url)
+ caption.name = caption.name.split(" - ")[0]
+ caption
+ end
+ captions ||= [] of Caption
+ @captions = captions
+ return @captions.as(Array(Caption))
+ end
+
+ def description
+ description = info
+ .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText")
+ .try &.as_s || ""
+ end
+
+ # TODO
+ def description=(value : String)
+ @description = value
+ end
+
+ def description_html
+ info["descriptionHtml"]?.try &.as_s || ""
+ end
+
+ def description_html=(value : String)
+ info["descriptionHtml"] = JSON::Any.new(value)
+ end
+
+ def short_description
+ info["shortDescription"]?.try &.as_s? || ""
end
def hls_manifest_url : String?
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
end
- def dash_manifest_url : String?
- raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s
- return nil if raw_dash_url.nil?
+ def dash_manifest_url
+ info.dig?("streamingData", "dashManifestUrl").try &.as_s
+ end
- # Use manifest v5 parameter to reduce file size
- # See https://github.com/iv-org/invidious/issues/4186
- dash_url = URI.parse(raw_dash_url)
- dash_query = dash_url.query || ""
-
- if dash_query.empty?
- dash_url.path = "#{dash_url.path}/mpd_version/5"
- else
- dash_url.query = "#{dash_query}&mpd_version=5"
- end
-
- return dash_url.to_s
+ def genre : String
+ info["genre"]?.try &.as_s || ""
end
def genre_url : String?
- info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
+ info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
end
- def vr? : Bool?
- return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
+ def license : String?
+ info["license"]?.try &.as_s
+ end
+
+ def is_family_friendly : Bool
+ info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false
+ end
+
+ def is_vr : Bool?
+ projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
+ return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
end
def projection_type : String?
@@ -180,118 +811,290 @@ struct Video
def reason : String?
info["reason"]?.try &.as_s
end
+end
- def music : Array(VideoMusic)
- info["music"].as_a.map { |music_json|
- VideoMusic.new(
- music_json["song"].as_s,
- music_json["album"].as_s,
- music_json["artist"].as_s,
- music_json["license"].as_s
- )
- }
+struct Caption
+ property name
+ property language_code
+ property base_url
+
+ getter name : String
+ getter language_code : String
+ getter base_url : String
+
+ setter name
+
+ def initialize(@name, @language_code, @base_url)
+ end
+end
+
+class VideoRedirect < Exception
+ property video_id : String
+
+ def initialize(@video_id)
+ end
+end
+
+# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
+# The former is preferred as it has more videos in it. The second has
+# the same 11 first entries as the compact rendered.
+#
+# TODO: "compactRadioRenderer" (Mix) and
+# TODO: Use a proper struct/class instead of a hacky JSON object
+def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
+ return nil if !related["videoId"]?
+
+ # The compact renderer has video length in seconds, where the end
+ # screen rendered has a full text version ("42:40")
+ length = related["lengthInSeconds"]?.try &.as_i.to_s
+ length ||= related.dig?("lengthText", "simpleText").try do |box|
+ decode_length_seconds(box.as_s).to_s
end
- # Macros defining getters/setters for various types of data
+ # Both have "short", so the "long" option shouldn't be required
+ channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
+ .try &.dig?("runs", 0)
- private macro getset_string(name)
- # Return {{name.stringify}} from `info`
- def {{name.id.underscore}} : String
- return info[{{name.stringify}}]?.try &.as_s || ""
- end
+ author = channel_info.try &.dig?("text")
+ author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
- # Update {{name.stringify}} into `info`
- def {{name.id.underscore}}=(value : String)
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
+ ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
+ # "4,088,033 views", only available on compact renderer
+ # and when video is not a livestream
+ view_count = related.dig?("viewCountText", "simpleText")
+ .try &.as_s.gsub(/\D/, "")
+
+ short_view_count = related.try do |r|
+ HelperExtractors.get_short_view_count(r).to_s
end
- private macro getset_string_array(name)
- # Return {{name.stringify}} from `info`
- def {{name.id.underscore}} : Array(String)
- return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String
- end
+ LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
- # Update {{name.stringify}} into `info`
- def {{name.id.underscore}}=(value : Array(String))
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
+ # TODO: when refactoring video types, make a struct for related videos
+ # or reuse an existing type, if that fits.
+ return {
+ "id" => related["videoId"],
+ "title" => related["title"]["simpleText"],
+ "author" => author || JSON::Any.new(""),
+ "ucid" => JSON::Any.new(ucid || ""),
+ "length_seconds" => JSON::Any.new(length || "0"),
+ "view_count" => JSON::Any.new(view_count || "0"),
+ "short_view_count" => JSON::Any.new(short_view_count || "0"),
+ "author_verified" => JSON::Any.new(author_verified),
+ }
+end
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
+def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil)
+ # Init client config for the API
+ client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
+ if context_screen == "embed"
+ client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
end
- {% for op, type in {i32: Int32, i64: Int64} %}
- private macro getset_{{op}}(name)
- def \{{name.id.underscore}} : {{type}}
- return info[\{{name.stringify}}]?.try &.as_i64.to_{{op}} || 0_{{op}}
+ # Fetch data from the player endpoint
+ player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
+
+ playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
+
+ if playability_status != "OK"
+ subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
+ reason = subreason.try &.[]?("simpleText").try &.as_s
+ reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
+ reason ||= player_response.dig("playabilityStatus", "reason").as_s
+
+ # Stop here if video is not a scheduled livestream
+ if playability_status != "LIVE_STREAM_OFFLINE"
+ return {
+ "reason" => JSON::Any.new(reason),
+ }
+ end
+ elsif video_id != player_response.dig("videoDetails", "videoId")
+ # YouTube may return a different video player response than expected.
+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+ raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)")
+ else
+ reason = nil
+ end
+
+ # Don't fetch the next endpoint if the video is unavailable.
+ if {"OK", "LIVE_STREAM_OFFLINE"}.any?(playability_status)
+ next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
+ player_response = player_response.merge(next_response)
+ end
+
+ params = parse_video_info(video_id, player_response)
+ params["reason"] = JSON::Any.new(reason) if reason
+
+ # Fetch the video streams using an Android client in order to get the decrypted URLs and
+ # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs:
+ # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+ if reason.nil?
+ if context_screen == "embed"
+ client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
+ else
+ client_config.client_type = YoutubeAPI::ClientType::Android
+ end
+ android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
+
+ # Sometimes, the video is available from the web client, but not on Android, so check
+ # that here, and fallback to the streaming data from the web client if needed.
+ # See: https://github.com/iv-org/invidious/issues/2549
+ if video_id != android_player.dig("videoDetails", "videoId")
+ # YouTube may return a different video player response than expected.
+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+ raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (ANDROID client)")
+ elsif android_player["playabilityStatus"]["status"] == "OK"
+ params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("")
+ else
+ params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("")
+ end
+ end
+
+ # TODO: clean that up
+ {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f|
+ params[f] = player_response[f] if player_response[f]?
+ end
+
+ return params
+end
+
+def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
+ # Top level elements
+
+ main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
+
+ raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
+
+ primary_results = main_results.dig?("results", "results", "contents")
+
+ raise BrokenTubeException.new("results") if !primary_results
+
+ video_primary_renderer = primary_results
+ .as_a.find(&.["videoPrimaryInfoRenderer"]?)
+ .try &.["videoPrimaryInfoRenderer"]
+
+ video_secondary_renderer = primary_results
+ .as_a.find(&.["videoSecondaryInfoRenderer"]?)
+ .try &.["videoSecondaryInfoRenderer"]
+
+ raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
+ raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
+
+ # Related videos
+
+ LOGGER.debug("extract_video_info: parsing related videos...")
+
+ related = [] of JSON::Any
+
+ # Parse "compactVideoRenderer" items (under secondary results)
+ secondary_results = main_results
+ .dig?("secondaryResults", "secondaryResults", "results")
+ secondary_results.try &.as_a.each do |element|
+ if item = element["compactVideoRenderer"]?
+ related_video = parse_related_video(item)
+ related << JSON::Any.new(related_video) if related_video
+ end
+ end
+
+ # If nothing was found previously, fall back to end screen renderer
+ if related.empty?
+ # Container for "endScreenVideoRenderer" items
+ player_overlays = player_response.dig?(
+ "playerOverlays", "playerOverlayRenderer",
+ "endScreen", "watchNextEndScreenRenderer", "results"
+ )
+
+ player_overlays.try &.as_a.each do |element|
+ if item = element["endScreenVideoRenderer"]?
+ related_video = parse_related_video(item)
+ related << JSON::Any.new(related_video) if related_video
end
-
- def \{{name.id.underscore}}=(value : Int)
- info[\{{name.stringify}}] = JSON::Any.new(value.to_i64)
- end
-
- \{% if flag?(:debug_macros) %} \{{debug}} \{% end %}
end
- {% end %}
-
- private macro getset_bool(name)
- # Return {{name.stringify}} from `info`
- def {{name.id.underscore}} : Bool
- return info[{{name.stringify}}]?.try &.as_bool || false
- end
-
- # Update {{name.stringify}} into `info`
- def {{name.id.underscore}}=(value : Bool)
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
-
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- # Macro to generate ? and = accessor methods for attributes in `info`
- private macro predicate_bool(method_name, name)
- # Return {{name.stringify}} from `info`
- def {{method_name.id.underscore}}? : Bool
- return info[{{name.stringify}}]?.try &.as_bool || false
- end
+ # Likes
- # Update {{name.stringify}} into `info`
- def {{method_name.id.underscore}}=(value : Bool)
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
+ toplevel_buttons = video_primary_renderer
+ .try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
+ if toplevel_buttons
+ likes_button = toplevel_buttons.as_a
+ .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE")
+ .try &.["toggleButtonRenderer"]
+
+ if likes_button
+ likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?)
+ .try &.dig?("accessibility", "accessibilityData", "label")
+ likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
+
+ LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
+ LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
+ end
end
- # Method definitions, using the macros above
+ # Description
- getset_string author
- getset_string authorThumbnail
- getset_string description
- getset_string descriptionHtml
- getset_string genre
- getset_string genreUcid
- getset_string license
- getset_string shortDescription
- getset_string subCountText
- getset_string title
- getset_string ucid
+ short_description = player_response.dig?("videoDetails", "shortDescription")
- getset_string_array allowedRegions
- getset_string_array keywords
+ description_html = video_secondary_renderer.try &.dig?("description", "runs")
+ .try &.as_a.try { |t| content_to_comment_html(t, video_id) }
- getset_i32 lengthSeconds
- getset_i64 likes
- getset_i64 views
+ # Video metadata
- # TODO: Make predicate_bool the default as to adhere to Crystal conventions
- getset_bool allowRatings
- getset_bool authorVerified
- getset_bool isFamilyFriendly
- getset_bool isListed
- predicate_bool upcoming, isUpcoming
+ metadata = video_secondary_renderer
+ .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
+ .try &.as_a
+
+ genre = player_response.dig?("microformat", "playerMicroformatRenderer", "category")
+ genre_ucid = nil
+ license = nil
+
+ metadata.try &.each do |row|
+ metadata_title = row.dig?("metadataRowRenderer", "title", "simpleText").try &.as_s
+ contents = row.dig?("metadataRowRenderer", "contents", 0)
+
+ if metadata_title == "Category"
+ contents = contents.try &.dig?("runs", 0)
+
+ genre = contents.try &.["text"]?
+ genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId")
+ elsif metadata_title == "License"
+ license = contents.try &.dig?("runs", 0, "text")
+ elsif metadata_title == "Licensed to YouTube by"
+ license = contents.try &.["simpleText"]?
+ end
+ end
+
+ # Author infos
+
+ if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
+ author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
+ author_verified = has_verified_badge?(author_info["badges"]?)
+
+ subs_text = author_info["subscriberCountText"]?
+ .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
+ .try &.as_s.split(" ", 2)[0]
+ end
+
+ # Return data
+
+ params = {
+ "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+ "relatedVideos" => JSON::Any.new(related),
+ "likes" => JSON::Any.new(likes || 0_i64),
+ "dislikes" => JSON::Any.new(0_i64),
+ "descriptionHtml" => JSON::Any.new(description_html || ""),
+ "genre" => JSON::Any.new(genre.try &.as_s || ""),
+ "genreUrl" => JSON::Any.new(nil),
+ "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
+ "license" => JSON::Any.new(license.try &.as_s || ""),
+ "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
+ "authorVerified" => JSON::Any.new(author_verified),
+ "subCountText" => JSON::Any.new(subs_text || "-"),
+ }
+
+ return params
end
def get_video(id, refresh = true, region = nil, force_refresh = false)
@@ -301,8 +1104,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
if (refresh &&
(Time.utc - video.updated > 10.minutes) ||
(video.premiere_timestamp.try &.< Time.utc)) ||
- force_refresh ||
- video.schema_version != Video::SCHEMA_VERSION # cache control
+ force_refresh
begin
video = fetch_video(id, region)
Invidious::Database::Videos.update(video)
@@ -326,12 +1128,31 @@ end
def fetch_video(id, region)
info = extract_video_info(video_id: id)
+ allowed_regions = info
+ .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
+ .try &.as_a.map &.as_s || [] of String
+
+ # Check for region-blocks
+ if info["reason"]?.try &.as_s.includes?("your country")
+ bypass_regions = PROXY_LIST.keys & allowed_regions
+ if !bypass_regions.empty?
+ region = bypass_regions[rand(bypass_regions.size)]
+ region_info = extract_video_info(video_id: id, proxy_region: region)
+ region_info["region"] = JSON::Any.new(region) if region
+ info = region_info if !region_info["reason"]?
+ end
+ end
+
+ # Try to fetch video info using an embedded client
+ if info["reason"]?
+ embed_info = extract_video_info(video_id: id, context_screen: "embed")
+ info = embed_info if !embed_info["reason"]?
+ end
+
if reason = info["reason"]?
if reason == "Video unavailable"
raise NotFoundException.new(reason.as_s || "")
- elsif !reason.as_s.starts_with? "Premieres"
- # dont error when it's a premiere.
- # we already parsed most of the data and display the premiere date
+ else
raise InfoException.new(reason.as_s || "")
end
end
@@ -345,6 +1166,10 @@ def fetch_video(id, region)
return video
end
+def itag_to_metadata?(itag : JSON::Any)
+ return VIDEO_FORMATS[itag.to_s]?
+end
+
def process_continuation(query, plid, id)
continuation = nil
if plid
@@ -359,6 +1184,135 @@ def process_continuation(query, plid, id)
continuation
end
+def process_video_params(query, preferences)
+ annotations = query["iv_load_policy"]?.try &.to_i?
+ autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ comments = query["comments"]?.try &.split(",").map(&.downcase)
+ continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ player_style = query["player_style"]?
+ preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
+ quality = query["quality"]?
+ quality_dash = query["quality_dash"]?
+ region = query["region"]?
+ related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ speed = query["speed"]?.try &.rchop("x").to_f?
+ video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ volume = query["volume"]?.try &.to_i?
+ vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+
+ if preferences
+ # region ||= preferences.region
+ annotations ||= preferences.annotations.to_unsafe
+ autoplay ||= preferences.autoplay.to_unsafe
+ comments ||= preferences.comments
+ continue ||= preferences.continue.to_unsafe
+ continue_autoplay ||= preferences.continue_autoplay.to_unsafe
+ listen ||= preferences.listen.to_unsafe
+ local ||= preferences.local.to_unsafe
+ player_style ||= preferences.player_style
+ preferred_captions ||= preferences.captions
+ quality ||= preferences.quality
+ quality_dash ||= preferences.quality_dash
+ related_videos ||= preferences.related_videos.to_unsafe
+ speed ||= preferences.speed
+ video_loop ||= preferences.video_loop.to_unsafe
+ extend_desc ||= preferences.extend_desc.to_unsafe
+ volume ||= preferences.volume
+ vr_mode ||= preferences.vr_mode.to_unsafe
+ save_player_pos ||= preferences.save_player_pos.to_unsafe
+ end
+
+ annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
+ autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
+ comments ||= CONFIG.default_user_preferences.comments
+ continue ||= CONFIG.default_user_preferences.continue.to_unsafe
+ continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
+ listen ||= CONFIG.default_user_preferences.listen.to_unsafe
+ local ||= CONFIG.default_user_preferences.local.to_unsafe
+ player_style ||= CONFIG.default_user_preferences.player_style
+ preferred_captions ||= CONFIG.default_user_preferences.captions
+ quality ||= CONFIG.default_user_preferences.quality
+ quality_dash ||= CONFIG.default_user_preferences.quality_dash
+ related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
+ speed ||= CONFIG.default_user_preferences.speed
+ video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
+ extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
+ volume ||= CONFIG.default_user_preferences.volume
+ vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
+ save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
+
+ annotations = annotations == 1
+ autoplay = autoplay == 1
+ continue = continue == 1
+ continue_autoplay = continue_autoplay == 1
+ listen = listen == 1
+ local = local == 1
+ related_videos = related_videos == 1
+ video_loop = video_loop == 1
+ extend_desc = extend_desc == 1
+ vr_mode = vr_mode == 1
+ save_player_pos = save_player_pos == 1
+
+ if CONFIG.disabled?("dash") && quality == "dash"
+ quality = "high"
+ end
+
+ if CONFIG.disabled?("local") && local
+ local = false
+ end
+
+ if start = query["t"]? || query["time_continue"]? || query["start"]?
+ video_start = decode_time(start)
+ end
+ video_start ||= 0
+
+ if query["end"]?
+ video_end = decode_time(query["end"])
+ end
+ video_end ||= -1
+
+ raw = query["raw"]?.try &.to_i?
+ raw ||= 0
+ raw = raw == 1
+
+ controls = query["controls"]?.try &.to_i?
+ controls ||= 1
+ controls = controls >= 1
+
+ params = VideoPreferences.new({
+ annotations: annotations,
+ autoplay: autoplay,
+ comments: comments,
+ continue: continue,
+ continue_autoplay: continue_autoplay,
+ controls: controls,
+ listen: listen,
+ local: local,
+ player_style: player_style,
+ preferred_captions: preferred_captions,
+ quality: quality,
+ quality_dash: quality_dash,
+ raw: raw,
+ region: region,
+ related_videos: related_videos,
+ speed: speed,
+ video_end: video_end,
+ video_loop: video_loop,
+ extend_desc: extend_desc,
+ video_start: video_start,
+ volume: volume,
+ vr_mode: vr_mode,
+ save_player_pos: save_player_pos,
+ })
+
+ return params
+end
+
def build_thumbnails(id)
return {
{host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"},
@@ -372,3 +1326,34 @@ def build_thumbnails(id)
{host: HOST_URL, height: 90, width: 120, name: "end", url: "3"},
}
end
+
+def generate_thumbnails(json, id)
+ json.array do
+ build_thumbnails(id).each do |thumbnail|
+ json.object do
+ json.field "quality", thumbnail[:name]
+ json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
+ json.field "width", thumbnail[:width]
+ json.field "height", thumbnail[:height]
+ end
+ end
+ end
+end
+
+def generate_storyboards(json, id, storyboards)
+ json.array do
+ storyboards.each do |storyboard|
+ json.object do
+ json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}"
+ json.field "templateUrl", storyboard[:url]
+ json.field "width", storyboard[:width]
+ json.field "height", storyboard[:height]
+ json.field "count", storyboard[:count]
+ json.field "interval", storyboard[:interval]
+ json.field "storyboardWidth", storyboard[:storyboard_width]
+ json.field "storyboardHeight", storyboard[:storyboard_height]
+ json.field "storyboardCount", storyboard[:storyboard_count]
+ end
+ end
+ end
+end
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
deleted file mode 100644
index c811cfe1..00000000
--- a/src/invidious/videos/caption.cr
+++ /dev/null
@@ -1,224 +0,0 @@
-require "json"
-
-module Invidious::Videos
- module Captions
- struct Metadata
- property name : String
- property language_code : String
- property base_url : String
-
- property auto_generated : Bool
-
- def initialize(@name, @language_code, @base_url, @auto_generated)
- end
-
- # Parse the JSON structure from Youtube
- def self.from_yt_json(container : JSON::Any) : Array(Captions::Metadata)
- caption_tracks = container
- .dig?("playerCaptionsTracklistRenderer", "captionTracks")
- .try &.as_a
-
- captions_list = [] of Captions::Metadata
- return captions_list if caption_tracks.nil?
-
- caption_tracks.each do |caption|
- name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
- name = name.to_s.split(" - ")[0]
-
- language_code = caption["languageCode"].to_s
- base_url = caption["baseUrl"].to_s
-
- auto_generated = (caption["kind"]? == "asr")
-
- captions_list << Captions::Metadata.new(name, language_code, base_url, auto_generated)
- end
-
- return captions_list
- end
-
- def timedtext_to_vtt(timedtext : String, tlang = nil) : String
- # In the future, we could just directly work with the url. This is more of a POC
- cues = [] of XML::Node
- tree = XML.parse(timedtext)
- tree = tree.children.first
-
- tree.children.each do |item|
- if item.name == "body"
- item.children.each do |cue|
- if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n")
- cues << cue
- end
- end
- break
- end
- end
-
- settings_field = {
- "Kind" => "captions",
- "Language" => "#{tlang || @language_code}",
- }
-
- result = WebVTT.build(settings_field) do |vtt|
- cues.each_with_index do |node, i|
- start_time = node["t"].to_f.milliseconds
-
- duration = node["d"]?.try &.to_f.milliseconds
-
- duration ||= start_time
-
- if cues.size > i + 1
- end_time = cues[i + 1]["t"].to_f.milliseconds
- else
- end_time = start_time + duration
- end
-
- text = String.build do |io|
- node.children.each do |s|
- io << s.content
- end
- end
-
- vtt.cue(start_time, end_time, text)
- end
- end
-
- return result
- end
- end
-
- # List of all caption languages available on Youtube.
- LANGUAGES = {
- "",
- "English",
- "English (auto-generated)",
- "English (United Kingdom)",
- "English (United States)",
- "Afrikaans",
- "Albanian",
- "Amharic",
- "Arabic",
- "Armenian",
- "Azerbaijani",
- "Bangla",
- "Basque",
- "Belarusian",
- "Bosnian",
- "Bulgarian",
- "Burmese",
- "Cantonese (Hong Kong)",
- "Catalan",
- "Cebuano",
- "Chinese",
- "Chinese (China)",
- "Chinese (Hong Kong)",
- "Chinese (Simplified)",
- "Chinese (Taiwan)",
- "Chinese (Traditional)",
- "Corsican",
- "Croatian",
- "Czech",
- "Danish",
- "Dutch",
- "Dutch (auto-generated)",
- "Esperanto",
- "Estonian",
- "Filipino",
- "Filipino (auto-generated)",
- "Finnish",
- "French",
- "French (auto-generated)",
- "Galician",
- "Georgian",
- "German",
- "German (auto-generated)",
- "Greek",
- "Gujarati",
- "Haitian Creole",
- "Hausa",
- "Hawaiian",
- "Hebrew",
- "Hindi",
- "Hmong",
- "Hungarian",
- "Icelandic",
- "Igbo",
- "Indonesian",
- "Indonesian (auto-generated)",
- "Interlingue",
- "Irish",
- "Italian",
- "Italian (auto-generated)",
- "Japanese",
- "Japanese (auto-generated)",
- "Javanese",
- "Kannada",
- "Kazakh",
- "Khmer",
- "Korean",
- "Korean (auto-generated)",
- "Kurdish",
- "Kyrgyz",
- "Lao",
- "Latin",
- "Latvian",
- "Lithuanian",
- "Luxembourgish",
- "Macedonian",
- "Malagasy",
- "Malay",
- "Malayalam",
- "Maltese",
- "Maori",
- "Marathi",
- "Mongolian",
- "Nepali",
- "Norwegian Bokmål",
- "Nyanja",
- "Pashto",
- "Persian",
- "Polish",
- "Portuguese",
- "Portuguese (auto-generated)",
- "Portuguese (Brazil)",
- "Punjabi",
- "Romanian",
- "Russian",
- "Russian (auto-generated)",
- "Samoan",
- "Scottish Gaelic",
- "Serbian",
- "Shona",
- "Sindhi",
- "Sinhala",
- "Slovak",
- "Slovenian",
- "Somali",
- "Southern Sotho",
- "Spanish",
- "Spanish (auto-generated)",
- "Spanish (Latin America)",
- "Spanish (Mexico)",
- "Spanish (Spain)",
- "Sundanese",
- "Swahili",
- "Swedish",
- "Tajik",
- "Tamil",
- "Telugu",
- "Thai",
- "Turkish",
- "Turkish (auto-generated)",
- "Ukrainian",
- "Urdu",
- "Uzbek",
- "Vietnamese",
- "Vietnamese (auto-generated)",
- "Welsh",
- "Western Frisian",
- "Xhosa",
- "Yiddish",
- "Yoruba",
- "Zulu",
- }
- end
-end
diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr
deleted file mode 100644
index 29c57182..00000000
--- a/src/invidious/videos/clip.cr
+++ /dev/null
@@ -1,22 +0,0 @@
-require "json"
-
-# returns start_time, end_time and clip_title
-def parse_clip_parameters(params) : {Float64?, Float64?, String?}
- decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
- .try { |i| Base64.decode(i) }
- .try { |i| IO::Memory.new(i) }
- .try { |i| Protodec::Any.parse(i) }
-
- start_time = decoded_protobuf
- .try(&.["50:0:embedded"]["2:1:varint"].as_i64)
- .try { |i| i/1000 }
-
- end_time = decoded_protobuf
- .try(&.["50:0:embedded"]["3:2:varint"].as_i64)
- .try { |i| i/1000 }
-
- clip_title = decoded_protobuf
- .try(&.["50:0:embedded"]["4:3:string"].as_s)
-
- return start_time, end_time, clip_title
-end
diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr
deleted file mode 100644
index 1371bebb..00000000
--- a/src/invidious/videos/description.cr
+++ /dev/null
@@ -1,82 +0,0 @@
-require "json"
-require "uri"
-
-private def copy_string(str : String::Builder, iter : Iterator, count : Int) : Int
- copied = 0
- while copied < count
- cp = iter.next
- break if cp.is_a?(Iterator::Stop)
-
- if cp == 0x26 # Ampersand (&)
- str << "&"
- elsif cp == 0x27 # Single quote (')
- str << "'"
- elsif cp == 0x22 # Double quote (")
- str << """
- elsif cp == 0x3C # Less-than (<)
- str << "<"
- elsif cp == 0x3E # Greater than (>)
- str << ">"
- else
- str << cp.chr
- end
-
- # A codepoint from the SMP counts twice
- copied += 1 if cp > 0xFFFF
- copied += 1
- end
-
- return copied
-end
-
-def parse_description(desc, video_id : String) : String?
- return "" if desc.nil?
-
- content = desc["content"].as_s
- return "" if content.empty?
-
- commands = desc["commandRuns"]?.try &.as_a
- if commands.nil?
- # Slightly faster than HTML.escape, as we're only doing one pass on
- # the string instead of five for the standard library
- return String.build do |str|
- copy_string(str, content.each_codepoint, content.size)
- end
- end
-
- # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints
- # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are
- # automatically decoded by the JSON parser. It means that we need to count
- # copied byte in a special manner, preventing the use of regular string copy.
- iter = content.each_codepoint
-
- index = 0
-
- return String.build do |str|
- commands.each do |command|
- cmd_start = command["startIndex"].as_i
- cmd_length = command["length"].as_i
-
- # Copy the text chunk between this command and the previous if needed.
- length = cmd_start - index
- index += copy_string(str, iter, length)
-
- # We need to copy the command's text using the iterator
- # and the special function defined above.
- cmd_content = String.build(cmd_length) do |str2|
- copy_string(str2, iter, cmd_length)
- end
-
- link = cmd_content
- if on_tap = command.dig?("onTap", "innertubeCommand")
- link = parse_link_endpoint(on_tap, cmd_content, video_id)
- end
- str << link
- index += cmd_length
- end
-
- # Copy the end of the string (past the last command).
- remaining_length = content.size - index
- copy_string(str, iter, remaining_length) if remaining_length > 0
- end
-end
diff --git a/src/invidious/videos/formats.cr b/src/invidious/videos/formats.cr
deleted file mode 100644
index e98e7257..00000000
--- a/src/invidious/videos/formats.cr
+++ /dev/null
@@ -1,116 +0,0 @@
-module Invidious::Videos::Formats
- def self.itag_to_metadata?(itag : JSON::Any)
- return FORMATS[itag.to_s]?
- end
-
- # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
- private FORMATS = {
- "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
- "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
- "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
- "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
- "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
- "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
-
- "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
- "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
-
- # 3D videos
- "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
-
- # Apple HTTP Live Streaming
- "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
- "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
- "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
-
- # DASH mp4 video
- "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
- "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
- "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
- "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
- "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
- "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
- "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
- "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
- "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
- "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
- "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
- "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
-
- # Dash mp4 audio
- "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
- "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
- "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
- "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
- "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
- "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
- "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
-
- # Dash webm
- "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
- "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
- "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
- "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
- "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
- "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
- # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
- "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
- "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
- "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
-
- # Dash webm audio
- "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
- "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
-
- # Dash webm audio with opus inside
- "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
- "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
- "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
-
- # av01 video only formats sometimes served with "unknown" codecs
- "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
- "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
- "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
- "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
- }
-end
diff --git a/src/invidious/videos/music.cr b/src/invidious/videos/music.cr
deleted file mode 100644
index 08d88a3e..00000000
--- a/src/invidious/videos/music.cr
+++ /dev/null
@@ -1,13 +0,0 @@
-require "json"
-
-struct VideoMusic
- include JSON::Serializable
-
- property song : String
- property album : String
- property artist : String
- property license : String
-
- def initialize(@song : String, @album : String, @artist : String, @license : String)
- end
-end
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
deleted file mode 100644
index feb58440..00000000
--- a/src/invidious/videos/parser.cr
+++ /dev/null
@@ -1,504 +0,0 @@
-require "json"
-
-# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
-# The former is preferred as it has more videos in it. The second has
-# the same 11 first entries as the compact rendered.
-#
-# TODO: "compactRadioRenderer" (Mix) and
-# TODO: Use a proper struct/class instead of a hacky JSON object
-def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
- return nil if !related["videoId"]?
-
- # The compact renderer has video length in seconds, where the end
- # screen rendered has a full text version ("42:40")
- length = related["lengthInSeconds"]?.try &.as_i.to_s
- length ||= related.dig?("lengthText", "simpleText").try do |box|
- decode_length_seconds(box.as_s).to_s
- end
-
- # Both have "short", so the "long" option shouldn't be required
- channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
- .try &.dig?("runs", 0)
-
- author = channel_info.try &.dig?("text")
- author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
-
- ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
-
- # "4,088,033 views", only available on compact renderer
- # and when video is not a livestream
- view_count = related.dig?("viewCountText", "simpleText")
- .try &.as_s.gsub(/\D/, "")
-
- short_view_count = related.try do |r|
- HelperExtractors.get_short_view_count(r).to_s
- end
-
- LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
-
- if published_time_text = related["publishedTimeText"]?
- decoded_time = decode_date(published_time_text["simpleText"].to_s)
- published = decoded_time.to_rfc3339.to_s
- else
- published = nil
- end
-
- # TODO: when refactoring video types, make a struct for related videos
- # or reuse an existing type, if that fits.
- return {
- "id" => related["videoId"],
- "title" => related["title"]["simpleText"],
- "author" => author || JSON::Any.new(""),
- "ucid" => JSON::Any.new(ucid || ""),
- "length_seconds" => JSON::Any.new(length || "0"),
- "view_count" => JSON::Any.new(view_count || "0"),
- "short_view_count" => JSON::Any.new(short_view_count || "0"),
- "author_verified" => JSON::Any.new(author_verified),
- "published" => JSON::Any.new(published || ""),
- }
-end
-
-def extract_video_info(video_id : String)
- # Init client config for the API
- client_config = YoutubeAPI::ClientConfig.new
-
- # Fetch data from the player endpoint
- player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
-
- playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
-
- if playability_status != "OK"
- subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
- reason = subreason.try &.[]?("simpleText").try &.as_s
- reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
- reason ||= player_response.dig("playabilityStatus", "reason").as_s
-
- # Stop here if video is not a scheduled livestream or
- # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
- if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
- playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
- return {
- "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
- "reason" => JSON::Any.new(reason),
- }
- end
- elsif video_id != player_response.dig?("videoDetails", "videoId")
- # YouTube may return a different video player response than expected.
- # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
- # Line to be reverted if one day we solve the video not available issue.
-
- # Although technically not a call to /videoplayback the fact that YouTube is returning the
- # wrong video means that we should count it as a failure.
- get_playback_statistic()["totalRequests"] += 1
-
- return {
- "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
- "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."),
- }
- else
- reason = nil
- end
-
- # Don't fetch the next endpoint if the video is unavailable.
- if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
- next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
- player_response = player_response.merge(next_response)
- end
-
- params = parse_video_info(video_id, player_response)
- params["reason"] = JSON::Any.new(reason) if reason
-
- if !CONFIG.invidious_companion.present?
- if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
- players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile}
-
- players_fallback.each do |player_fallback|
- client_config.client_type = player_fallback
-
- next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
-
- if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
- streaming_data = player_response["streamingData"].as_h
- streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
- player_response["streamingData"] = JSON::Any.new(streaming_data)
- break
- end
- rescue InfoException
- next LOGGER.warn("Failed to fetch streams with #{player_fallback}")
- end
- end
-
- # Seems like video page can still render even without playable streams.
- # its better than nothing.
- #
- # # Were we able to find playable video streams?
- # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- # # No :(
- # end
- end
-
- {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
- params[f] = player_response[f] if player_response[f]?
- end
-
- # Convert URLs, if those are present
- if streaming_data = player_response["streamingData"]?
- %w[formats adaptiveFormats].each do |key|
- streaming_data.as_h[key]?.try &.as_a.each do |format|
- format.as_h["url"] = JSON::Any.new(convert_url(format))
- end
- end
-
- params["streamingData"] = streaming_data
- end
-
- # Data structure version, for cache control
- params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
-
- return params
-end
-
-def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
- LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
- response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
-
- playability_status = response["playabilityStatus"]["status"]
- LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
-
- if id != response.dig?("videoDetails", "videoId")
- # YouTube may return a different video player response than expected.
- # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
- raise InfoException.new(
- "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
- )
- elsif playability_status == "OK"
- return response
- else
- return nil
- end
-end
-
-def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
- # Top level elements
-
- main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
-
- raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
-
- # Primary results are not available on Music videos
- # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725
- if primary_results = main_results.dig?("results", "results", "contents")
- video_primary_renderer = primary_results
- .as_a.find(&.["videoPrimaryInfoRenderer"]?)
- .try &.["videoPrimaryInfoRenderer"]
-
- video_secondary_renderer = primary_results
- .as_a.find(&.["videoSecondaryInfoRenderer"]?)
- .try &.["videoSecondaryInfoRenderer"]
-
- raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
- raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
- end
-
- video_details = player_response.dig?("videoDetails")
- if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
- microformat = {} of String => JSON::Any
- end
-
- raise BrokenTubeException.new("videoDetails") if !video_details
-
- # Basic video infos
-
- title = video_details["title"]?.try &.as_s
-
- # We have to try to extract viewCount from videoPrimaryInfoRenderer first,
- # then from videoDetails, as the latter is "0" for livestreams (we want
- # to get the amount of viewers watching).
- views_txt = extract_text(
- video_primary_renderer
- .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount")
- )
- views_txt ||= video_details["viewCount"]?.try &.as_s || ""
- views = views_txt.gsub(/\D/, "").to_i64?
-
- length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
- .try &.as_s.to_i64
-
- published = microformat["publishDate"]?
- .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
-
- premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
- .try { |t| Time.parse_rfc3339(t.as_s) }
-
- premiere_timestamp ||= player_response.dig?(
- "playabilityStatus", "liveStreamability",
- "liveStreamabilityRenderer", "offlineSlate",
- "liveStreamOfflineSlateRenderer", "scheduledStartTime"
- )
- .try &.as_s.to_i64
- .try { |t| Time.unix(t) }
-
- live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
- .try &.as_bool
- live_now ||= video_details.dig?("isLive").try &.as_bool || false
-
- post_live_dvr = video_details.dig?("isPostLiveDvr")
- .try &.as_bool || false
-
- # Extra video infos
-
- allowed_regions = microformat["availableCountries"]?
- .try &.as_a.map &.as_s || [] of String
-
- allow_ratings = video_details["allowRatings"]?.try &.as_bool
- family_friendly = microformat["isFamilySafe"]?.try &.as_bool
- is_listed = video_details["isCrawlable"]?.try &.as_bool
- is_upcoming = video_details["isUpcoming"]?.try &.as_bool
-
- keywords = video_details["keywords"]?
- .try &.as_a.map &.as_s || [] of String
-
- # Related videos
-
- LOGGER.debug("extract_video_info: parsing related videos...")
-
- related = [] of JSON::Any
-
- # Parse "compactVideoRenderer" items (under secondary results)
- secondary_results = main_results
- .dig?("secondaryResults", "secondaryResults", "results")
- secondary_results.try &.as_a.each do |element|
- if item = element["compactVideoRenderer"]?
- related_video = parse_related_video(item)
- related << JSON::Any.new(related_video) if related_video
- end
- end
-
- # If nothing was found previously, fall back to end screen renderer
- if related.empty?
- # Container for "endScreenVideoRenderer" items
- player_overlays = player_response.dig?(
- "playerOverlays", "playerOverlayRenderer",
- "endScreen", "watchNextEndScreenRenderer", "results"
- )
-
- player_overlays.try &.as_a.each do |element|
- if item = element["endScreenVideoRenderer"]?
- related_video = parse_related_video(item)
- related << JSON::Any.new(related_video) if related_video
- end
- end
- end
-
- # Likes
-
- toplevel_buttons = video_primary_renderer
- .try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
-
- if toplevel_buttons
- # New Format as of december 2023
- likes_button = toplevel_buttons.dig?(0,
- "segmentedLikeDislikeButtonViewModel",
- "likeButtonViewModel",
- "likeButtonViewModel",
- "toggleButtonViewModel",
- "toggleButtonViewModel",
- "defaultButtonViewModel",
- "buttonViewModel"
- )
-
- likes_button ||= toplevel_buttons.try &.as_a
- .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE")
- .try &.["toggleButtonRenderer"]
-
- # New format as of september 2022
- likes_button ||= toplevel_buttons.try &.as_a
- .find(&.["segmentedLikeDislikeButtonRenderer"]?)
- .try &.dig?(
- "segmentedLikeDislikeButtonRenderer",
- "likeButton", "toggleButtonRenderer"
- )
-
- if likes_button
- likes_txt = likes_button.dig?("accessibilityText")
- # Note: The like count from `toggledText` is off by one, as it would
- # represent the new like count in the event where the user clicks on "like".
- likes_txt ||= (likes_button["defaultText"]? || likes_button["toggledText"]?)
- .try &.dig?("accessibility", "accessibilityData", "label")
- likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
-
- LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
- LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
- end
- end
-
- # Description
-
- description = microformat.dig?("description", "simpleText").try &.as_s || ""
- short_description = player_response.dig?("videoDetails", "shortDescription")
-
- # description_html = video_secondary_renderer.try &.dig?("description", "runs")
- # .try &.as_a.try { |t| content_to_comment_html(t, video_id) }
-
- description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"), video_id)
-
- # Video metadata
-
- metadata = video_secondary_renderer
- .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
- .try &.as_a
-
- genre = microformat["category"]?
- genre_ucid = nil
- license = nil
-
- metadata.try &.each do |row|
- metadata_title = extract_text(row.dig?("metadataRowRenderer", "title"))
- contents = row.dig?("metadataRowRenderer", "contents", 0)
-
- if metadata_title == "Category"
- contents = contents.try &.dig?("runs", 0)
-
- genre = contents.try &.["text"]?
- genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId")
- elsif metadata_title == "License"
- license = contents.try &.dig?("runs", 0, "text")
- elsif metadata_title == "Licensed to YouTube by"
- license = contents.try &.["simpleText"]?
- end
- end
-
- # Music section
-
- music_list = [] of VideoMusic
- music_desclist = player_response.dig?(
- "engagementPanels", 1, "engagementPanelSectionListRenderer",
- "content", "structuredDescriptionContentRenderer", "items", 2,
- "videoDescriptionMusicSectionRenderer", "carouselLockups"
- )
-
- music_desclist.try &.as_a.each do |music_desc|
- artist = nil
- album = nil
- music_license = nil
-
- # Used when the video has multiple songs
- if song_title = music_desc.dig?("carouselLockupRenderer", "videoLockup", "compactVideoRenderer", "title")
- # "simpleText" for plain text / "runs" when song has a link
- song = song_title["simpleText"]? || song_title.dig?("runs", 0, "text")
-
- # some videos can have empty tracks. See: https://www.youtube.com/watch?v=eBGIQ7ZuuiU
- next if !song
- end
-
- music_desc.dig?("carouselLockupRenderer", "infoRows").try &.as_a.each do |desc|
- desc_title = extract_text(desc.dig?("infoRowRenderer", "title"))
- if desc_title == "ARTIST"
- artist = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
- elsif desc_title == "SONG"
- song = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
- elsif desc_title == "ALBUM"
- album = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
- elsif desc_title == "LICENSES"
- music_license = extract_text(desc.dig?("infoRowRenderer", "expandedMetadata"))
- end
- end
- music_list << VideoMusic.new(song.to_s, album.to_s, artist.to_s, music_license.to_s)
- end
-
- # Author infos
-
- author = video_details["author"]?.try &.as_s
- ucid = video_details["channelId"]?.try &.as_s
-
- if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
- author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
- author_verified = has_verified_badge?(author_info["badges"]?)
-
- subs_text = author_info["subscriberCountText"]?
- .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
- .try &.as_s.split(" ", 2)[0]
- end
-
- # Return data
-
- if live_now
- video_type = VideoType::Livestream
- elsif !premiere_timestamp.nil?
- video_type = VideoType::Scheduled
- published = premiere_timestamp || Time.utc
- else
- video_type = VideoType::Video
- end
-
- params = {
- "videoType" => JSON::Any.new(video_type.to_s),
- # Basic video infos
- "title" => JSON::Any.new(title || ""),
- "views" => JSON::Any.new(views || 0_i64),
- "likes" => JSON::Any.new(likes || 0_i64),
- "lengthSeconds" => JSON::Any.new(length_txt || 0_i64),
- "published" => JSON::Any.new(published.to_rfc3339),
- # Extra video infos
- "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
- "allowRatings" => JSON::Any.new(allow_ratings || false),
- "isFamilyFriendly" => JSON::Any.new(family_friendly || false),
- "isListed" => JSON::Any.new(is_listed || false),
- "isUpcoming" => JSON::Any.new(is_upcoming || false),
- "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
- "isPostLiveDvr" => JSON::Any.new(post_live_dvr),
- # Related videos
- "relatedVideos" => JSON::Any.new(related),
- # Description
- "description" => JSON::Any.new(description || ""),
- "descriptionHtml" => JSON::Any.new(description_html || ""),
- "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
- # Video metadata
- "genre" => JSON::Any.new(genre.try &.as_s || ""),
- "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
- "license" => JSON::Any.new(license.try &.as_s || ""),
- # Music section
- "music" => JSON.parse(music_list.to_json),
- # Author infos
- "author" => JSON::Any.new(author || ""),
- "ucid" => JSON::Any.new(ucid || ""),
- "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
- "authorVerified" => JSON::Any.new(author_verified || false),
- "subCountText" => JSON::Any.new(subs_text || "-"),
- }
-
- return params
-end
-
-private def convert_url(fmt)
- if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
- sp = cfr["sp"]
- url = URI.parse(cfr["url"])
- params = url.query_params
-
- LOGGER.debug("convert_url: Decoding '#{cfr}'")
-
- unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
- params[sp] = unsig if unsig
- else
- url = URI.parse(fmt["url"].as_s)
- params = url.query_params
- end
-
- n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
- params["n"] = n if n
-
- if token = CONFIG.po_token
- params["pot"] = token
- end
-
- url.query_params = params
- LOGGER.trace("convert_url: new url is '#{url}'")
-
- return url.to_s
-rescue ex
- LOGGER.debug("convert_url: Error when parsing video URL")
- LOGGER.trace(ex.inspect_with_backtrace)
- return ""
-end
diff --git a/src/invidious/videos/regions.cr b/src/invidious/videos/regions.cr
deleted file mode 100644
index 575f8c25..00000000
--- a/src/invidious/videos/regions.cr
+++ /dev/null
@@ -1,27 +0,0 @@
-# List of geographical regions that Youtube recognizes.
-# This is used to determine if a video is either restricted to a list
-# of allowed regions (= whitelisted) or if it can't be watched in
-# a set of regions (= blacklisted).
-REGIONS = {
- "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT",
- "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI",
- "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY",
- "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN",
- "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM",
- "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK",
- "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL",
- "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM",
- "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR",
- "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN",
- "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS",
- "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK",
- "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW",
- "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP",
- "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM",
- "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW",
- "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM",
- "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF",
- "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW",
- "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI",
- "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW",
-}
diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr
deleted file mode 100644
index bd0eef59..00000000
--- a/src/invidious/videos/storyboard.cr
+++ /dev/null
@@ -1,122 +0,0 @@
-require "uri"
-require "http/params"
-
-module Invidious::Videos
- struct Storyboard
- # Template URL
- getter url : URI
- getter proxied_url : URI
-
- # Thumbnail parameters
- getter width : Int32
- getter height : Int32
- getter count : Int32
- getter interval : Int32
-
- # Image (storyboard) parameters
- getter rows : Int32
- getter columns : Int32
- getter images_count : Int32
-
- def initialize(
- *, @url, @width, @height, @count, @interval,
- @rows, @columns, @images_count,
- )
- authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]?
-
- @proxied_url = URI.parse(HOST_URL)
- @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}"
- @proxied_url.query = @url.query
- end
-
- # Parse the JSON structure from Youtube
- def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard)
- # Livestream storyboards are a bit different
- # TODO: document exactly how
- if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s
- return [Storyboard.new(
- url: URI.parse(storyboard.split("#")[0]),
- width: 106,
- height: 60,
- count: -1,
- interval: 5000,
- rows: 3,
- columns: 3,
- images_count: -1
- )]
- end
-
- # Split the storyboard string into chunks
- #
- # General format (whitespaces added for legibility):
- # https://i.ytimg.com/sb//storyboard3_L$L/$N.jpg?sqp=
- # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$
- # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$
- # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$
- #
- storyboards = container.dig?("playerStoryboardSpecRenderer", "spec")
- .try &.as_s.split("|")
-
- return [] of Storyboard if !storyboards
-
- # The base URL is the first chunk
- base_url = URI.parse(storyboards.shift)
-
- return storyboards.map_with_index do |sb, i|
- # Separate the different storyboard parameters:
- # width/height: respective dimensions, in pixels, of a single thumbnail
- # count: how many thumbnails are displayed across the full video
- # columns/rows: maximum amount of thumbnails that can be stuffed in a
- # single image, horizontally and vertically.
- # interval: interval between two thumbnails, in milliseconds
- # name: storyboard filename. Usually "M$M" or "default"
- # sigh: URL cryptographic signature
- width, height, count, columns, rows, interval, name, sigh = sb.split("#")
-
- width = width.to_i
- height = height.to_i
- count = count.to_i
- interval = interval.to_i
- columns = columns.to_i
- rows = rows.to_i
-
- # Copy base URL object, so that we can modify it
- url = base_url.dup
-
- # Add the signature to the URL
- params = url.query_params
- params["sigh"] = sigh
- url.query_params = params
-
- # Replace the template parts with what we have
- url.path = url.path.sub("$L", i).sub("$N", name)
-
- # This value represents the maximum amount of thumbnails that can fit
- # in a single image. The last image (or the only one for short videos)
- # will contain less thumbnails than that.
- thumbnails_per_image = columns * rows
-
- # This value represents the total amount of storyboards required to
- # hold all of the thumbnails. It can't be less than 1.
- images_count = (count / thumbnails_per_image).ceil.to_i
-
- # Compute the interval when needed (in general, that's only required
- # for the first "default" storyboard).
- if interval == 0
- interval = ((length_seconds / count) * 1_000).to_i
- end
-
- Storyboard.new(
- url: url,
- width: width,
- height: height,
- count: count,
- interval: interval,
- rows: rows,
- columns: columns,
- images_count: images_count,
- )
- end
- end
- end
-end
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
deleted file mode 100644
index ee1272d1..00000000
--- a/src/invidious/videos/transcript.cr
+++ /dev/null
@@ -1,161 +0,0 @@
-module Invidious::Videos
- # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video.
- # These lines can be categorized into two types: section headings and regular lines representing content from the video.
- struct Transcript
- # Types
- record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- alias TranscriptLine = HeadingLine | RegularLine
-
- property lines : Array(TranscriptLine)
-
- property language_code : String
- property auto_generated : Bool
-
- # User friendly label for the current transcript.
- # Example: "English (auto-generated)"
- property label : String
-
- # Initializes a new Transcript struct with the contents and associated metadata describing it
- def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String)
- end
-
- # Generates a protobuf string to fetch the requested transcript from YouTube
- def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
- kind = auto_generated ? "asr" : ""
-
- object = {
- "1:0:string" => video_id,
-
- "2:base64" => {
- "1:string" => kind,
- "2:string" => language_code,
- "3:string" => "",
- },
-
- "3:varint" => 1_i64,
- "5:string" => "engagement-panel-searchable-transcript-search-panel",
- "6:varint" => 1_i64,
- "7:varint" => 1_i64,
- "8:varint" => 1_i64,
- }
-
- params = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- return params
- end
-
- # Constructs a Transcripts struct from the initial YouTube response
- def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool)
- transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
- "content", "transcriptSearchPanelRenderer")
-
- segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer")
-
- if !segment_list["initialSegments"]?
- raise NotFoundException.new("Requested transcript does not exist")
- end
-
- # Extract user-friendly label for the current transcript
-
- footer_language_menu = transcript_panel.dig?(
- "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems"
- )
-
- if footer_language_menu
- label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
- else
- label = language_code
- end
-
- # Extract transcript lines
-
- initial_segments = segment_list["initialSegments"].as_a
-
- lines = [] of TranscriptLine
-
- initial_segments.each do |line|
- if unpacked_line = line["transcriptSectionHeaderRenderer"]?
- line_type = HeadingLine
- else
- unpacked_line = line["transcriptSegmentRenderer"]
- line_type = RegularLine
- end
-
- start_ms = unpacked_line["startMs"].as_s.to_i.millisecond
- end_ms = unpacked_line["endMs"].as_s.to_i.millisecond
- text = extract_text(unpacked_line["snippet"]) || ""
-
- lines << line_type.new(start_ms, end_ms, text)
- end
-
- return Transcript.new(
- lines: lines,
- language_code: language_code,
- auto_generated: auto_generated,
- label: label
- )
- end
-
- # Converts transcript lines to a WebVTT file
- #
- # This is used within Invidious to replace subtitles
- # as to workaround YouTube's rate-limited timedtext endpoint.
- def to_vtt
- settings_field = {
- "Kind" => "captions",
- "Language" => @language_code,
- }
-
- vtt = WebVTT.build(settings_field) do |builder|
- @lines.each do |line|
- # Section headers are excluded from the VTT conversion as to
- # match the regular captions returned from YouTube as much as possible
- next if line.is_a? HeadingLine
-
- builder.cue(line.start_ms, line.end_ms, line.line)
- end
- end
-
- return vtt
- end
-
- def to_json(json : JSON::Builder)
- json.field "languageCode", @language_code
- json.field "autoGenerated", @auto_generated
- json.field "label", @label
- json.field "body" do
- json.array do
- @lines.each do |line|
- json.object do
- if line.is_a? HeadingLine
- json.field "type", "heading"
- else
- json.field "type", "regular"
- end
-
- json.field "startMs", line.start_ms.total_milliseconds
- json.field "endMs", line.end_ms.total_milliseconds
- json.field "line", line.line
- end
- end
- end
- end
- end
-
- def to_json
- JSON.build do |json|
- json.object do
- json.field "transcript" do
- json.object do
- to_json(json)
- end
- end
- end
- end
- end
- end
-end
diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr
deleted file mode 100644
index 48177bd8..00000000
--- a/src/invidious/videos/video_preferences.cr
+++ /dev/null
@@ -1,162 +0,0 @@
-struct VideoPreferences
- include JSON::Serializable
-
- property annotations : Bool
- property preload : Bool
- property autoplay : Bool
- property comments : Array(String)
- property continue : Bool
- property continue_autoplay : Bool
- property controls : Bool
- property listen : Bool
- property local : Bool
- property preferred_captions : Array(String)
- property player_style : String
- property quality : String
- property quality_dash : String
- property raw : Bool
- property region : String?
- property related_videos : Bool
- property speed : Float32 | Float64
- property video_end : Float64 | Int32
- property video_loop : Bool
- property extend_desc : Bool
- property video_start : Float64 | Int32
- property volume : Int32
- property vr_mode : Bool
- property save_player_pos : Bool
-end
-
-def process_video_params(query, preferences)
- annotations = query["iv_load_policy"]?.try &.to_i?
- preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- comments = query["comments"]?.try &.split(",").map(&.downcase)
- continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- player_style = query["player_style"]?
- preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
- quality = query["quality"]?
- quality_dash = query["quality_dash"]?
- region = query["region"]?
- related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- speed = query["speed"]?.try &.rchop("x").to_f?
- video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- volume = query["volume"]?.try &.to_i?
- vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
-
- if preferences
- # region ||= preferences.region
- annotations ||= preferences.annotations.to_unsafe
- preload ||= preferences.preload.to_unsafe
- autoplay ||= preferences.autoplay.to_unsafe
- comments ||= preferences.comments
- continue ||= preferences.continue.to_unsafe
- continue_autoplay ||= preferences.continue_autoplay.to_unsafe
- listen ||= preferences.listen.to_unsafe
- local ||= preferences.local.to_unsafe
- player_style ||= preferences.player_style
- preferred_captions ||= preferences.captions
- quality ||= preferences.quality
- quality_dash ||= preferences.quality_dash
- related_videos ||= preferences.related_videos.to_unsafe
- speed ||= preferences.speed
- video_loop ||= preferences.video_loop.to_unsafe
- extend_desc ||= preferences.extend_desc.to_unsafe
- volume ||= preferences.volume
- vr_mode ||= preferences.vr_mode.to_unsafe
- save_player_pos ||= preferences.save_player_pos.to_unsafe
- end
-
- annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
- preload ||= CONFIG.default_user_preferences.preload.to_unsafe
- autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
- comments ||= CONFIG.default_user_preferences.comments
- continue ||= CONFIG.default_user_preferences.continue.to_unsafe
- continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
- listen ||= CONFIG.default_user_preferences.listen.to_unsafe
- local ||= CONFIG.default_user_preferences.local.to_unsafe
- player_style ||= CONFIG.default_user_preferences.player_style
- preferred_captions ||= CONFIG.default_user_preferences.captions
- quality ||= CONFIG.default_user_preferences.quality
- quality_dash ||= CONFIG.default_user_preferences.quality_dash
- related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
- speed ||= CONFIG.default_user_preferences.speed
- video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
- extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
- volume ||= CONFIG.default_user_preferences.volume
- vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
- save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
-
- annotations = annotations == 1
- preload = preload == 1
- autoplay = autoplay == 1
- continue = continue == 1
- continue_autoplay = continue_autoplay == 1
- listen = listen == 1
- local = local == 1
- related_videos = related_videos == 1
- video_loop = video_loop == 1
- extend_desc = extend_desc == 1
- vr_mode = vr_mode == 1
- save_player_pos = save_player_pos == 1
-
- if CONFIG.disabled?("dash") && quality == "dash"
- quality = "high"
- end
-
- if CONFIG.disabled?("local") && local
- local = false
- end
-
- if start = query["t"]? || query["time_continue"]? || query["start"]?
- video_start = decode_time(start)
- end
- video_start ||= 0
-
- if query["end"]?
- video_end = decode_time(query["end"])
- end
- video_end ||= -1
-
- raw = query["raw"]?.try &.to_i?
- raw ||= 0
- raw = raw == 1
-
- controls = query["controls"]?.try &.to_i?
- controls ||= 1
- controls = controls >= 1
-
- params = VideoPreferences.new({
- annotations: annotations,
- preload: preload,
- autoplay: autoplay,
- comments: comments,
- continue: continue,
- continue_autoplay: continue_autoplay,
- controls: controls,
- listen: listen,
- local: local,
- player_style: player_style,
- preferred_captions: preferred_captions,
- quality: quality,
- quality_dash: quality_dash,
- raw: raw,
- region: region,
- related_videos: related_videos,
- speed: speed,
- video_end: video_end,
- video_loop: video_loop,
- extend_desc: extend_desc,
- video_start: video_start,
- volume: volume,
- vr_mode: vr_mode,
- save_player_pos: save_player_pos,
- })
-
- return params
-end
diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr
index 6aea82ae..22870317 100644
--- a/src/invidious/views/add_playlist_items.ecr
+++ b/src/invidious/views/add_playlist_items.ecr
@@ -31,5 +31,31 @@
+
+ <% videos.each_slice(4) do |slice| %>
+ <% slice.each do |item| %>
+ <%= rendered "components/item" %>
+ <% end %>
+ <% end %>
+
-<%= rendered "components/items_paginated" %>
+<% if query %>
+ <%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
+
+
+ <% if query.page > 1 %>
+
+ <%= translate(locale, "Previous page") %>
+
+ <% end %>
+
+
+
+ <% if videos.size >= 20 %>
+
+ <%= translate(locale, "Next page") %>
+
+ <% end %>
+
+
+<% end %>
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 686de6bd..92f81ee4 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,59 +1,116 @@
-<%-
- ucid = channel.ucid
- author = HTML.escape(channel.author)
- channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
-
- relative_url =
- case selected_tab
- when .shorts? then "/channel/#{ucid}/shorts"
- when .streams? then "/channel/#{ucid}/streams"
- when .playlists? then "/channel/#{ucid}/playlists"
- when .channels? then "/channel/#{ucid}/channels"
- when .podcasts? then "/channel/#{ucid}/podcasts"
- when .releases? then "/channel/#{ucid}/releases"
- when .courses? then "/channel/#{ucid}/courses"
- else
- "/channel/#{ucid}"
- end
-
- youtube_url = "https://www.youtube.com#{relative_url}"
- redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-
- page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
- base_url: relative_url,
- ctoken: next_continuation,
- first_page: continuation.nil?,
- params: env.params.query,
- )
-%>
+<% ucid = channel.ucid %>
+<% author = HTML.escape(channel.author) %>
<% content_for "header" do %>
-<%- if selected_tab.videos? -%>
-
-
-
-
-
-
-
-
-
-
-
-
-<%- end -%>
-
-
-
-
<%= author %> - Invidious
+
<% end %>
-<%= rendered "components/channel_info" %>
+<% if channel.banner %>
+
+ ">
+
+
+
+
+
+<% end %>
+
+
+
+
+
+ <%= channel.description_html %>
+
+
+
+
+ <% sub_count_text = number_to_short_text(channel.sub_count) %>
+ <%= rendered "components/subscribe_widget" %>
+
+
+
+
+ <%= translate(locale, "View channel on YouTube") %>
+
+ <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
+ "><%= translate(locale, "Switch Invidious Instance") %>
+ <% else %>
+ <%= translate(locale, "Switch Invidious Instance") %>
+ <% end %>
+
+ <% if !channel.auto_generated %>
+
+ <%= translate(locale, "Videos") %>
+
+ <% end %>
+
+ <% if channel.auto_generated %>
+ <%= translate(locale, "Playlists") %>
+ <% else %>
+ <%= translate(locale, "Playlists") %>
+ <% end %>
+
+
+ <% if channel.tabs.includes? "community" %>
+ <%= translate(locale, "Community") %>
+ <% end %>
+
+
+
+
+
+ <% sort_options.each do |sort| %>
+
+ <% if sort_by == sort %>
+ <%= translate(locale, sort) %>
+ <% else %>
+
+ <%= translate(locale, sort) %>
+
+ <% end %>
+
+ <% end %>
+
+
+
+
+<% items.each do |item| %>
+ <%= rendered "components/item" %>
+<% end %>
+
-<%= rendered "components/items_paginated" %>
+
+
+ <% if page > 1 %>
+ &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <%= translate(locale, "Previous page") %>
+
+ <% end %>
+
+
+
+ <% if count == 60 %>
+ &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <%= translate(locale, "Next page") %>
+
+ <% end %>
+
+
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 132e636c..3bc29e55 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,21 +1,71 @@
-<%-
- ucid = channel.ucid
- author = HTML.escape(channel.author)
- channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
-
- relative_url = "/channel/#{ucid}/community"
- youtube_url = "https://www.youtube.com#{relative_url}"
- redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-
- selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Posts
--%>
+<% ucid = channel.ucid %>
+<% author = HTML.escape(channel.author) %>
<% content_for "header" do %>
-
<%= author %> - Invidious
<% end %>
-<%= rendered "components/channel_info" %>
+<% if channel.banner %>
+
+ ">
+
+
+
+
+
+<% end %>
+
+
+
+
+
+ <%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>
+
+
+
+
+ <% sub_count_text = number_to_short_text(channel.sub_count) %>
+ <%= rendered "components/subscribe_widget" %>
+
+
+
+
+ <%= translate(locale, "View channel on YouTube") %>
+
+ <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
+ "><%= translate(locale, "Switch Invidious Instance") %>
+ <% else %>
+ <%= translate(locale, "Switch Invidious Instance") %>
+ <% end %>
+
+ <% if !channel.auto_generated %>
+
+ <% end %>
+
+
+ <% if channel.tabs.includes? "community" %>
+ <%= translate(locale, "Community") %>
+ <% end %>
+
+
+
+
@@ -26,8 +76,8 @@
<%= error_message %>
<% else %>
-
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index c8db207c..9bb73136 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -42,6 +42,11 @@ module Invidious::Routes::Account sid = sid.as(String) token = env.params.body["csrf_token"]? + # We don't store passwords for Google accounts + if !user.password + return error_template(400, "Cannot change password for Google accounts") + end + begin validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex @@ -49,11 +54,11 @@ module Invidious::Routes::Account end password = env.params.body["password"]? - if password.nil? || password.empty? + if !password return error_template(401, "Password is a required field") end - new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } + new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 return error_template(400, "New passwords must match") @@ -198,7 +203,7 @@ module Invidious::Routes::Account referer = get_referer(env) if !user - return env.redirect "/login?referer=#{URI.encode_path_segment(env.request.resource)}" + return env.redirect referer end user = user.as(User) @@ -240,7 +245,7 @@ module Invidious::Routes::Account return error_template(400, ex) end - scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } + scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? @@ -257,7 +262,6 @@ module Invidious::Routes::Account end query["token"] = access_token - query["username"] = URI.encode_path_segment(user.email) url.query = query.to_s env.redirect url.to_s @@ -328,9 +332,17 @@ module Invidious::Routes::Account end end - case action = env.params.query["action"]? - when "revoke_token" - session = env.params.query["session"] + if env.params.query["action_revoke_token"]? + action = "action_revoke_token" + else + return env.redirect referer + end + + session = env.params.query["session"]? + session ||= "" + + case action + when .starts_with? "action_revoke_token" Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index c27caad7..bfb8a377 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -8,17 +8,14 @@ module Invidious::Routes::API::Manifest id = env.params.url["id"] region = env.params.query["region"]? - if CONFIG.invidious_companion.present? - invidious_companion = CONFIG.invidious_companion.sample - return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}" - end - # Since some implementations create playlists based on resolution regardless of different codecs, # we can opt to only add a source to a representation if it has a unique height within that representation unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } begin video = get_video(id, region: region) + rescue ex : VideoRedirect + return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex : NotFoundException haltf env, status_code: 404 rescue ex @@ -26,27 +23,28 @@ module Invidious::Routes::API::Manifest end if dashmpd = video.dash_manifest_url - response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) + manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body - if response.status_code != 200 - haltf env, status_code: response.status_code - end + manifest = manifest.gsub(/" + + user_hash = lookup_results[0][2] + + if token = env.params.body["token"]? + answer = env.params.body["answer"]? + captcha = {token, answer} + else + captcha = nil + end + + challenge_req = { + user_hash, nil, 1, nil, + {1, nil, nil, nil, + {password, captcha, true}, + }, + {nil, nil, + {2, 1, nil, 1, + "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", + nil, [] of String, 4}, + 1, + {nil, nil, [] of String}, + nil, nil, nil, true, + }, + }.to_json + + traceback << "Getting challenge..." + + response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req)) + headers = response.cookies.add_request_headers(headers) + challenge_results = JSON.parse(response.body[5..-1]) + + traceback << "done, returned #{response.status_code}.
" + + headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) + + if challenge_results[0][3]?.try &.== 7 + return error_template(423, "Account has temporarily been disabled") + end + + if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s + account_type = "google" + captcha_type = "image" + prompt = nil + tfa = tfa_code + captcha = {tokens: [token], question: ""} + + return templated "user/login" + end + + if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" + return error_template(401, "Incorrect password") + end + + prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]? + if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type + traceback << "Handling prompt #{prompt_type}.
" + case prompt_type + when "TWO_STEP_VERIFICATION" + prompt_type = 2 + else # "LOGIN_CHALLENGE" + prompt_type = 4 + end + + # Prefer Authenticator app and SMS over unsupported protocols + if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2 + tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0] + + traceback << "Selecting challenge #{tfa[8]}..." + select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json + + tl = challenge_results[1][2] + + tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body + tfa = tfa[5..-1] + tfa = JSON.parse(tfa)[0][-1] + + traceback << "done.
" + else + traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.
" + tfa = challenge_results[0][-1][0][0] + end + + if tfa[5] == "QUOTA_EXCEEDED" + return error_template(423, "Quota exceeded, try again in a few hours") + end + + if !tfa_code + account_type = "google" + captcha_type = "image" + + case tfa[8] + when 6, 9 + prompt = "Google verification code" + when 12 + prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" + when 15 + prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" + else + prompt = "Google verification code" + end + + tfa = nil + captcha = nil + return templated "user/login" + end + + tl = challenge_results[1][2] + + request_type = tfa[8] + case request_type + when 6 # Authenticator app + tfa_req = { + user_hash, nil, 2, nil, + {6, nil, nil, nil, nil, + {tfa_code, false}, + }, + }.to_json + when 9 # Voice or text message + tfa_req = { + user_hash, nil, 2, nil, + {9, nil, nil, nil, nil, nil, nil, nil, + {nil, tfa_code, false, 2}, + }, + }.to_json + when 12 # Recovery email + tfa_req = { + user_hash, nil, 4, nil, + {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + {tfa_code}, + }, + }.to_json + when 15 # Security question + tfa_req = { + user_hash, nil, 5, nil, + {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + {tfa_code}, + }, + }.to_json + else + return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.") + end + + traceback << "Submitting challenge..." + + response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req)) + headers = response.cookies.add_request_headers(headers) + challenge_results = JSON.parse(response.body[5..-1]) + + if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") || + (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT") + return error_template(401, "Invalid TFA code") + end + + traceback << "done.
" + end + + traceback << "Logging in..." + + location = URI.parse(challenge_results[0][-1][2].to_s) + cookies = HTTP::Cookies.from_client_headers(headers) + + headers.delete("Content-Type") + headers.delete("Google-Accounts-XSRF") + + loop do + if !location || location.path == "/ManageAccount" + break + end + + # Occasionally there will be a second page after login confirming + # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle. + + if location.path.starts_with? "/b/0/SmsAuthInterstitial" + traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." + end + + login = client.get(location.request_target, headers) + + headers = login.cookies.add_request_headers(headers) + location = login.headers["Location"]?.try { |u| URI.parse(u) } + end + + cookies = HTTP::Cookies.from_client_headers(headers) + sid = cookies["SID"]?.try &.value + if !sid + raise "Couldn't get SID." + end + + user, sid = get_user(sid, headers) + + # We are now logged in + traceback << "done.
" + + host = URI.parse(env.request.headers["Host"]).host + + cookies.each do |cookie| + cookie.secure = Invidious::User::Cookies::SECURE + + if cookie.extension + cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host) + cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "") + end + env.response.cookies << cookie + end + + if env.request.cookies["PREFS"]? + user.preferences = env.get("preferences").as(Preferences) + Invidious::Database::Users.update_preferences(user) + + cookie = env.request.cookies["PREFS"] + cookie.expires = Time.utc(1990, 1, 1) + env.response.cookies << cookie + end + + env.redirect referer + rescue ex + traceback.rewind + # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.") + error_message = %(#{ex.message}
Traceback:
#{traceback.gets_to_end}
)
+ return error_template(500, error_message)
+ end
when "invidious"
- if email.nil? || email.empty?
+ if !email
return error_template(401, "User ID is a required field")
end
- if password.nil? || password.empty?
+ if !password
return error_template(401, "Password is a required field")
end
user = Invidious::Database::Users.select(email: email)
if user
+ if !user.password
+ return error_template(400, "Please sign in using 'Log in with Google'")
+ end
+
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
Invidious::Database::SessionIDs.insert(sid, email)
@@ -85,14 +356,36 @@ module Invidious::Routes::Login
password = password.byte_slice(0, 55)
if CONFIG.captcha_enabled
+ captcha_type = env.params.body["captcha_type"]?
answer = env.params.body["answer"]?
+ change_type = env.params.body["change_type"]?
- account_type = "invidious"
- captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ if !captcha_type || change_type
+ if change_type
+ captcha_type = change_type
+ end
+ captcha_type ||= "image"
+
+ account_type = "invidious"
+ tfa = false
+ prompt = ""
+
+ if captcha_type == "image"
+ captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ else
+ captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
+ end
+
+ return templated "user/login"
+ end
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
- if answer
+ answer ||= ""
+ captcha_type ||= "image"
+
+ case captcha_type
+ when "image"
answer = answer.lstrip('0')
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
@@ -101,8 +394,27 @@ module Invidious::Routes::Login
rescue ex
return error_template(400, ex)
end
- else
- return templated "user/login"
+ else # "text"
+ answer = Digest::MD5.hexdigest(answer.downcase.strip)
+
+ if tokens.empty?
+ return error_template(500, "Erroneous CAPTCHA")
+ end
+
+ found_valid_captcha = false
+ error_exception = Exception.new
+ tokens.each do |tok|
+ begin
+ validate_request(tok, answer, env.request, HMAC_KEY, locale)
+ found_valid_captcha = true
+ rescue ex
+ error_exception = ex
+ end
+ end
+
+ if !found_valid_captcha
+ return error_template(500, error_exception)
+ end
end
end
@@ -169,4 +481,11 @@ module Invidious::Routes::Login
env.redirect referer
end
+
+ def self.captcha(env)
+ headers = HTTP::Headers{":authority" => "accounts.google.com"}
+ response = YT_POOL.client &.get(env.request.resource, headers)
+ env.response.headers["Content-Type"] = response.headers["Content-Type"]
+ response.body
+ end
end
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index 0b868755..d6bd9571 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -40,21 +40,7 @@ module Invidious::Routes::Misc
def self.cross_instance_redirect(env)
referer = get_referer(env)
-
- instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
- # Filter out the current instance
- other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
-
- if other_available_instances.empty?
- # If the current instance is the only one, use the redirect URL as fallback
- instance_url = "redirect.invidious.io"
- else
- # Select other random instance
- # Sample returns an array
- # Instances are packaged as {region, domain} in the instance list
- instance_url = other_available_instances.sample(1)[0][1]
- end
-
+ instance_url = fetch_random_instance
env.redirect "https://#{instance_url}#{referer}"
end
end
diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr
index 8922b740..272a3dc7 100644
--- a/src/invidious/routes/notifications.cr
+++ b/src/invidious/routes/notifications.cr
@@ -24,6 +24,50 @@ module Invidious::Routes::Notifications
user = user.as(User)
+ if !user.password
+ channel_req = {} of String => String
+
+ channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true"
+ channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || ""
+ channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true"
+
+ channel_req.reject! { |k, v| v != "true" && v != "false" }
+
+ headers = HTTP::Headers.new
+ headers["Cookie"] = env.request.headers["Cookie"]
+
+ html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
+
+ cookies = HTTP::Cookies.from_client_headers(headers)
+ html.cookies.each do |cookie|
+ if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
+ if cookies[cookie.name]?
+ cookies[cookie.name] = cookie
+ else
+ cookies << cookie
+ end
+ end
+ end
+ headers = cookies.add_request_headers(headers)
+
+ if match = html.body.match(/'XSRF_TOKEN': "(?#{ex.message}") end @@ -432,13 +431,6 @@ module Invidious::Routes::Playlists env.set "remove_playlist_items", plid end - # Pagination - page_nav_html = Frontend::Pagination.nav_numeric(locale, - base_url: "/playlist?list=#{playlist.id}", - current_page: page, - show_next: (page_count != 1 && page < page_count) - ) - templated "playlist" end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 39ca77c0..570cba69 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -27,10 +27,6 @@ module Invidious::Routes::PreferencesRoute annotations_subscribed ||= "off" annotations_subscribed = annotations_subscribed == "on" - preload = env.params.body["preload"]?.try &.as(String) - preload ||= "off" - preload = preload == "on" - autoplay = env.params.body["autoplay"]?.try &.as(String) autoplay ||= "off" autoplay = autoplay == "on" @@ -148,7 +144,6 @@ module Invidious::Routes::PreferencesRoute preferences = Preferences.from_json({ annotations: annotations, annotations_subscribed: annotations_subscribed, - preload: preload, autoplay: autoplay, captions: captions, comments: comments, @@ -219,7 +214,7 @@ module Invidious::Routes::PreferencesRoute statistics_enabled ||= "off" CONFIG.statistics_enabled = statistics_enabled == "on" - CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence + CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String) File.write("config/config.yml", CONFIG.to_yaml) end @@ -315,24 +310,6 @@ module Invidious::Routes::PreferencesRoute response: error_template(415, "Invalid subscription file uploaded") ) end - when "import_youtube_pl" - filename = part.filename || "" - success = Invidious::User::Import.from_youtube_pl(user, body, filename, type) - - if !success - haltf(env, status_code: 415, - response: error_template(415, "Invalid playlist file uploaded") - ) - end - when "import_youtube_wh" - filename = part.filename || "" - success = Invidious::User::Import.from_youtube_wh(user, body, filename, type) - - if !success - haltf(env, status_code: 415, - response: error_template(415, "Invalid watch history file uploaded") - ) - end when "import_freetube" Invidious::User::Import.from_freetube(user, body) when "import_newpipe_subscriptions" diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index b195c7b3..2a9705cf 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -51,39 +51,21 @@ module Invidious::Routes::Search else user = env.get? "user" - # An URL was copy/pasted in the search box. - # Redirect the user to the appropriate page. - if query.url? - return env.redirect UrlSanitizer.process(query.text).to_s - end - begin - if user - items = query.process(user.as(User)) - else - items = query.process - end + videos = query.process rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex return error_template(500, ex) end + params = query.to_http_params + url_prev_page = "/search?#{params}&page=#{query.page - 1}" + url_next_page = "/search?#{params}&page=#{query.page + 1}" + redirect_url = Invidious::Frontend::Misc.redirect_url(env) - # Pagination - page_nav_html = Frontend::Pagination.nav_numeric(locale, - base_url: "/search?#{query.to_http_params}", - current_page: query.page, - show_next: (items.size >= 20) - ) - - if query.type == Invidious::Search::Query::Type::Channel - env.set "search", "channel:#{query.channel} #{query.text}" - else - env.set "search", query.text - end - + env.set "search", query.text templated "search" end end @@ -105,18 +87,16 @@ module Invidious::Routes::Search end begin - items = Invidious::Hashtag.fetch(hashtag, page) + videos = Invidious::Hashtag.fetch(hashtag, page) rescue ex return error_template(500, ex) end - # Pagination + params = env.params.query.empty? ? "" : "&#{env.params.query}" + hashtag_encoded = URI.encode_www_form(hashtag, space_to_plus: false) - page_nav_html = Frontend::Pagination.nav_numeric(locale, - base_url: "/hashtag/#{hashtag_encoded}", - current_page: page, - show_next: (items.size >= 60) - ) + url_prev_page = "/hashtag/#{hashtag_encoded}?page=#{page - 1}#{params}" + url_next_page = "/hashtag/#{hashtag_encoded}?page=#{page + 1}#{params}" templated "hashtag" end diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr index 1de655d2..7b1fa876 100644 --- a/src/invidious/routes/subscriptions.cr +++ b/src/invidious/routes/subscriptions.cr @@ -32,16 +32,29 @@ module Invidious::Routes::Subscriptions end end + if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 + action = "action_create_subscription_to_channel" + elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 + action = "action_remove_subscriptions" + else + return env.redirect referer + end + channel_id = env.params.query["c"]? channel_id ||= "" - case action = env.params.query["action"]? - when "create_subscription_to_channel" + if !user.password + # Sync subscriptions with YouTube + subscribe_ajax(channel_id, action, env.request.headers) + end + + case action + when "action_create_subscription_to_channel" if !user.subscriptions.includes? channel_id get_channel(channel_id) Invidious::Database::Users.subscribe_channel(user, channel_id) end - when "remove_subscriptions" + when "action_remove_subscriptions" Invidious::Database::Users.unsubscribe_channel(user, channel_id) else return error_json(400, "Unsupported action #{action}") @@ -69,6 +82,14 @@ module Invidious::Routes::Subscriptions user = user.as(User) sid = sid.as(String) + if !user.password + # Refresh account + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] + + user, sid = get_user(sid, headers) + end + action_takeout = env.params.query["action_takeout"]?.try &.to_i? action_takeout ||= 0 action_takeout = action_takeout == 1 @@ -83,8 +104,33 @@ module Invidious::Routes::Subscriptions if format == "json" env.response.content_type = "application/json" env.response.headers["content-disposition"] = "attachment" + playlists = Invidious::Database::Playlists.select_like_iv(user.email) - return Invidious::User::Export.to_invidious(user) + return JSON.build do |json| + json.object do + json.field "subscriptions", user.subscriptions + json.field "watch_history", user.watched + json.field "preferences", user.preferences + json.field "playlists" do + json.array do + playlists.each do |playlist| + json.object do + json.field "title", playlist.title + json.field "description", html_to_content(playlist.description_html) + json.field "privacy", playlist.privacy.to_s + json.field "videos" do + json.array do + Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id| + json.string video_id + end + end + end + end + end + end + end + end + end else env.response.content_type = "application/xml" env.response.headers["content-disposition"] = "attachment" diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 083087a9..560f9c19 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -21,7 +21,7 @@ module Invidious::Routes::VideoPlayback end # Sanity check, to avoid being used as an open proxy - if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/) + if !host.matches?(/[\w-]+.googlevideo.com/) return error_template(400, "Invalid \"host\" parameter.") end @@ -35,15 +35,7 @@ module Invidious::Routes::VideoPlayback end end - # See: https://github.com/iv-org/invidious/issues/3302 - range_header = env.request.headers["Range"]? - sq = query_params["sq"]? - if range_header.nil? && sq.nil? - range_for_head = query_params["range"]? || "0-640" - headers["Range"] = "bytes=#{range_for_head}" - end - - client = make_client(URI.parse(host), region, force_resolve: true) + client = make_client(URI.parse(host), region) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -58,7 +50,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region, force_resolve: true) + client = make_client(URI.parse(new_host), region) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -72,23 +64,15 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region, force_resolve: true) + client = make_client(URI.parse(host), region) rescue ex error = ex.message end end - # Remove the Range header added previously. - headers.delete("Range") if range_header.nil? - - playback_statistics = get_playback_statistic() - playback_statistics["totalRequests"] += 1 - if response.status_code >= 400 env.response.content_type = "text/plain" haltf env, response.status_code - else - playback_statistics["successfulRequests"] += 1 end if url.includes? "&file=seg.ts" @@ -107,8 +91,14 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? - url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) - return env.redirect url + location = URI.parse(location) + location = "#{location.request_target}&host=#{location.host}" + + if region + location += "®ion=#{region}" + end + + return env.redirect location end IO.copy(resp.body_io, env.response) @@ -132,7 +122,7 @@ module Invidious::Routes::VideoPlayback end # TODO: Record bytes written so we can restart after a chunk fails - loop do + while true if !range_end && content_length range_end = content_length end @@ -165,13 +155,10 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? - url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) + location = URI.parse(location) + location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" - if title = query_params["title"]? - url = "#{url}&title=#{URI.encode_www_form(title)}" - end - - env.redirect url + env.redirect location break end @@ -200,7 +187,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region, force_resolve: true) + client = make_client(URI.parse(host), region) end end @@ -257,11 +244,6 @@ module Invidious::Routes::VideoPlayback # YouTube /videoplayback links expire after 6 hours, # so we have a mechanism here to redirect to the latest version def self.latest_version(env) - if CONFIG.invidious_companion.present? - invidious_companion = CONFIG.invidious_companion.sample - return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}" - end - id = env.params.query["id"]? itag = env.params.query["itag"]?.try &.to_i? @@ -270,7 +252,7 @@ module Invidious::Routes::VideoPlayback return error_template(400, "Invalid video ID") end - if !itag.nil? && (itag <= 0 || itag >= 1000) + if itag.nil? || itag <= 0 || itag >= 1000 return error_template(400, "Invalid itag") end @@ -291,11 +273,7 @@ module Invidious::Routes::VideoPlayback return error_template(500, ex) end - if itag.nil? - fmt = video.fmt_stream[-1]? - else - fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } - end + fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } url = fmt.try &.["url"]?.try &.as_s if !url diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index e777b3f1..fe1d8e54 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -30,6 +30,14 @@ module Invidious::Routes::Watch return env.redirect "/" end + embed_link = "/embed/#{id}" + if env.params.query.size > 1 + embed_params = HTTP::Params.parse(env.params.query.to_s) + embed_params.delete_all("v") + embed_link += "?" + embed_link += embed_params.to_s + end + plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") continuation = process_continuation(env.params.query, plid, id) @@ -53,6 +61,8 @@ module Invidious::Routes::Watch begin video = get_video(id, region: params.region) + rescue ex : VideoRedirect + return env.redirect env.request.resource.gsub(id, ex.video_id) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) @@ -68,11 +78,11 @@ module Invidious::Routes::Watch end env.params.query.delete_all("iv_load_policy") - if watched && preferences.watch_history + if watched && preferences.watch_history && !watched.includes? id Invidious::Database::Users.mark_watched(user.as(User), id) end - if CONFIG.enable_user_notifications && notifications && notifications.includes? id + if notifications && notifications.includes? id Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) @@ -87,31 +97,31 @@ module Invidious::Routes::Watch if source == "youtube" begin - comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] rescue ex if preferences.comments[1] == "reddit" - comments, reddit_thread = Comments.fetch_reddit(id) - comment_html = Frontend::Comments.template_reddit(comments, locale) + comments, reddit_thread = fetch_reddit_comments(id) + comment_html = template_reddit_comments(comments, locale) - comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com") - comment_html = Comments.replace_links(comment_html) + comment_html = fill_links(comment_html, "https", "www.reddit.com") + comment_html = replace_links(comment_html) end end elsif source == "reddit" begin - comments, reddit_thread = Comments.fetch_reddit(id) - comment_html = Frontend::Comments.template_reddit(comments, locale) + comments, reddit_thread = fetch_reddit_comments(id) + comment_html = template_reddit_comments(comments, locale) - comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com") - comment_html = Comments.replace_links(comment_html) + comment_html = fill_links(comment_html, "https", "www.reddit.com") + comment_html = replace_links(comment_html) rescue ex if preferences.comments[1] == "youtube" - comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] end end end else - comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] end comment_html ||= "" @@ -121,12 +131,10 @@ module Invidious::Routes::Watch adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } end - # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } - video_streams = video.video_streams audio_streams = video.audio_streams @@ -192,14 +200,6 @@ module Invidious::Routes::Watch captions: video.captions ) - if CONFIG.invidious_companion.present? - invidious_companion = CONFIG.invidious_companion.sample - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion.public_url}") - .gsub("connect-src", "connect-src #{invidious_companion.public_url}") - end - templated "watch" end @@ -251,10 +251,20 @@ module Invidious::Routes::Watch end end - case action = env.params.query["action"]? - when "mark_watched" - Invidious::Database::Users.mark_watched(user, id) - when "mark_unwatched" + if env.params.query["action_mark_watched"]? + action = "action_mark_watched" + elsif env.params.query["action_mark_unwatched"]? + action = "action_mark_unwatched" + else + return env.redirect referer + end + + case action + when "action_mark_watched" + if !user.watched.includes? id + Invidious::Database::Users.mark_watched(user, id) + end + when "action_mark_unwatched" Invidious::Database::Users.mark_unwatched(user, id) else return error_json(400, "Unsupported action #{action}") @@ -277,12 +287,6 @@ module Invidious::Routes::Watch return error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") - if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s - start_time, end_time, _ = parse_clip_parameters(params) - env.params.query["start"] = start_time.to_s if start_time != nil - env.params.query["end"] = end_time.to_s if end_time != nil - end - return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else return error_template(404, "The requested clip doesn't exist") @@ -293,9 +297,6 @@ module Invidious::Routes::Watch if CONFIG.disabled?("downloads") return error_template(403, "Administrator has disabled this endpoint.") end - if CONFIG.invidious_companion.present? - return error_template(403, "Downloads should be routed through Companion when present") - end title = env.params.body["title"]? || "" video_id = env.params.body["id"]? || "" @@ -325,9 +326,10 @@ module Invidious::Routes::Watch env.params.query["label"] = URI.decode_www_form(label.as_s) return Invidious::Routes::API::V1::Videos.captions(env) - elsif itag = download_widget["itag"]?.try &.as_i.to_s + elsif itag = download_widget["itag"]?.try &.as_i # URL params specific to /latest_version env.params.query["id"] = video_id + env.params.query["itag"] = itag.to_s env.params.query["title"] = filename env.params.query["local"] = "true" diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 46b71f1f..f409f13c 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -37,9 +37,7 @@ module Invidious::Routing get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post - if CONFIG.enable_user_notifications - get "/modify_notifications", Routes::Notifications, :modify - end + get "/modify_notifications", Routes::Notifications, :modify {% end %} self.register_image_routes @@ -57,6 +55,7 @@ module Invidious::Routing get "/login", Routes::Login, :login_page post "/login", Routes::Login, :login post "/signout", Routes::Login, :signout + get "/Captcha", Routes::Login, :captcha # User preferences get "/preferences", Routes::PreferencesRoute, :show @@ -116,52 +115,29 @@ module Invidious::Routing get "/channel/:ucid", Routes::Channels, :home get "/channel/:ucid/home", Routes::Channels, :home get "/channel/:ucid/videos", Routes::Channels, :videos - get "/channel/:ucid/shorts", Routes::Channels, :shorts - get "/channel/:ucid/streams", Routes::Channels, :streams - get "/channel/:ucid/podcasts", Routes::Channels, :podcasts - get "/channel/:ucid/releases", Routes::Channels, :releases - get "/channel/:ucid/courses", Routes::Channels, :courses get "/channel/:ucid/playlists", Routes::Channels, :playlists get "/channel/:ucid/community", Routes::Channels, :community - get "/channel/:ucid/posts", Routes::Channels, :community - get "/channel/:ucid/channels", Routes::Channels, :channels get "/channel/:ucid/about", Routes::Channels, :about - get "/channel/:ucid/live", Routes::Channels, :live get "/user/:user/live", Routes::Channels, :live get "/c/:user/live", Routes::Channels, :live - get "/post/:id", Routes::Channels, :post - # Channel catch-all, to redirect future routes to the channel's home - # NOTE: defined last in order to be processed after the other routes - get "/channel/:ucid/*", Routes::Channels, :redirect_home - - # /c/LinusTechTips - get "/c/:user", Routes::Channels, :brand_redirect - get "/c/:user/:tab", Routes::Channels, :brand_redirect - - # /user/linustechtips (Not always the same as /c/) - get "/user/:user", Routes::Channels, :brand_redirect - get "/user/:user/:tab", Routes::Channels, :brand_redirect - - # /@LinusTechTips (Handle) - get "/@:user", Routes::Channels, :brand_redirect - get "/@:user/:tab", Routes::Channels, :brand_redirect - - # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow - get "/attribution_link", Routes::Channels, :brand_redirect - get "/attribution_link/:tab", Routes::Channels, :brand_redirect - - # /profile?user=linustechtips - get "/profile", Routes::Channels, :profile - get "/profile/*", Routes::Channels, :profile + ["", "/videos", "/playlists", "/community", "/about"].each do |path| + # /c/LinusTechTips + get "/c/:user#{path}", Routes::Channels, :brand_redirect + # /user/linustechtips | Not always the same as /c/ + get "/user/:user#{path}", Routes::Channels, :brand_redirect + # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow + get "/attribution_link#{path}", Routes::Channels, :brand_redirect + # /profile?user=linustechtips + get "/profile/#{path}", Routes::Channels, :profile + end end def register_watch_routes get "/watch", Routes::Watch, :handle post "/watch_ajax", Routes::Watch, :mark_watched get "/watch/:id", Routes::Watch, :redirect - get "/live/:id", Routes::Watch, :redirect get "/shorts/:id", Routes::Watch, :redirect get "/clip/:clip", Routes::Watch, :clip get "/w/:id", Routes::Watch, :redirect @@ -237,8 +213,6 @@ module Invidious::Routing get "/api/v1/captions/:id", {{namespace}}::Videos, :captions get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations get "/api/v1/comments/:id", {{namespace}}::Videos, :comments - get "/api/v1/clips/:id", {{namespace}}::Videos, :clips - get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts # Feeds get "/api/v1/trending", {{namespace}}::Feeds, :trending @@ -246,22 +220,10 @@ module Invidious::Routing # Channels get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home - get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest - get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos - get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts - get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams - get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts - get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases - get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses - get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists - get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community - get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community - get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels - get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search - - # Posts - get "/api/v1/post/:id", {{namespace}}::Channels, :post - get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments + {% for route in {"videos", "latest", "playlists", "community", "search"} %} + get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} + get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} + {% end %} # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect @@ -270,22 +232,17 @@ module Invidious::Routing # Search get "/api/v1/search", {{namespace}}::Search, :search get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions - get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag - # Authenticated + # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr + # + # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences - get "/api/v1/auth/export/invidious", {{namespace}}::Authenticated, :export_invidious - post "/api/v1/auth/import/invidious", {{namespace}}::Authenticated, :import_invidious - - get "/api/v1/auth/history", {{namespace}}::Authenticated, :get_history - post "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_watched - delete "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_unwatched - delete "/api/v1/auth/history", {{namespace}}::Authenticated, :clear_history - get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions @@ -303,17 +260,14 @@ module Invidious::Routing post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token - if CONFIG.enable_user_notifications - get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - end + get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications # Misc get "/api/v1/stats", {{namespace}}::Misc, :stats get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes - get "/api/v1/resolveurl", {{namespace}}::Misc, :resolve_url {% end %} end end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index bc2715cf..c2b5c758 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -75,7 +75,7 @@ module Invidious::Search @type : Type = Type::All, @duration : Duration = Duration::None, @features : Features = Features::None, - @sort : Sort = Sort::Relevance, + @sort : Sort = Sort::Relevance ) end @@ -300,9 +300,9 @@ module Invidious::Search object["9:varint"] = ((page - 1) * 20).to_i64 end - # Prevent censoring of self harm topics - # See https://github.com/iv-org/invidious/issues/4398 - object["30:varint"] = 1.to_i64 + # If the object is empty, return an empty string, + # otherwise encode to protobuf then to base64 + return "" if object.empty? return object .try { |i| Protodec::Any.cast_json(i) } diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index 25edb936..d1409c06 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -9,8 +9,7 @@ module Invidious::Search client_config = YoutubeAPI::ClientConfig.new(region: query.region) initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) - items, _ = extract_items(initial_data) - return items.reject!(Category) + return extract_items(initial_data) end # Search a youtube channel @@ -31,8 +30,17 @@ module Invidious::Search continuation = produce_channel_search_continuation(ucid, query.text, query.page) response_json = YoutubeAPI.browse(continuation) - items, _ = extract_items(response_json, "", ucid) - return items.reject!(Category) + continuation_items = response_json["onResponseReceivedActions"]? + .try &.[0]["appendContinuationItemsAction"]["continuationItems"] + + return [] of SearchItem if !continuation_items + + items = [] of SearchItem + continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| + extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } + end + + return items end # Search inside of user subscriptions diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index 94a92e23..24e79609 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -20,9 +20,6 @@ module Invidious::Search property region : String? property channel : String = "" - # Flag that indicates if the smart search features have been disabled. - @inhibit_ssf : Bool = false - # Return true if @raw_query is either `nil` or empty private def empty_raw_query? return @raw_query.empty? @@ -47,22 +44,14 @@ module Invidious::Search def initialize( params : HTTP::Params, @type : Type = Type::Regular, - @region : String? = nil, + @region : String? = nil ) # Get the raw search query string (common to all search types). In # Regular search mode, also look for the `search_query` URL parameter - _raw_query = params["q"]? - _raw_query ||= params["search_query"]? if @type.regular? - _raw_query ||= "" - - # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs. - @raw_query = _raw_query.strip - - # Check for smart features (ex: URL search) inhibitor (backslash). - # If inhibitor is present, remove it. - if @raw_query.starts_with?('\\') - @inhibit_ssf = true - @raw_query = @raw_query[1..] + if @type.regular? + @raw_query = params["q"]? || params["search_query"]? || "" + else + @raw_query = params["q"]? || "" end # Get the page number (also common to all search types) @@ -96,7 +85,7 @@ module Invidious::Search @filters = Filters.from_iv_params(params) @channel = params["channel"]? || "" - if @filters.default? && @raw_query.index(/\w:\w/) + if @filters.default? && @raw_query.includes?(':') # Parse legacy filters from query @filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query) else @@ -124,7 +113,7 @@ module Invidious::Search case @type when .regular?, .playlist? - items = Processors.regular(self) + items = unnest_items(Processors.regular(self)) # when .channel? items = Processors.channel(self) @@ -148,21 +137,25 @@ module Invidious::Search return params end - # Checks if the query is a standalone URL - def url? : Bool - # If the smart features have been inhibited, don't go further. - return false if @inhibit_ssf + # TODO: clean code + private def unnest_items(all_items) : Array(SearchItem) + items = [] of SearchItem - # Only supported in regular search mode - return false if !@type.regular? + # Light processing to flatten search results out of Categories. + # They should ideally be supported in the future. + all_items.each do |i| + if i.is_a? Category + i.contents.each do |nest_i| + if !nest_i.is_a? Video + items << nest_i + end + end + else + items << i + end + end - # If filters are present, that's a regular search - return false if !@filters.default? - - # Simple heuristics: domain name - return @raw_query.starts_with?( - /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\// - ) + return items end end end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index d14cde5d..1f957081 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -4,12 +4,11 @@ def fetch_trending(trending_type, region, locale) plid = nil - case trending_type.try &.downcase - when "music" + if trending_type == "Music" params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D" - when "gaming" + elsif trending_type == "Gaming" params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D" - when "movies" + elsif trending_type == "Movies" params = "4gIKGgh0cmFpbGVycw%3D%3D" else # Default params = "" @@ -17,26 +16,7 @@ def fetch_trending(trending_type, region, locale) client_config = YoutubeAPI::ClientConfig.new(region: region) initial_data = YoutubeAPI.browse("FEtrending", params: params, client_config: client_config) + trending = extract_videos(initial_data) - items, _ = extract_items(initial_data) - - extracted = [] of SearchItem - - deduplicate = items.size > 1 - - items.each do |itm| - if itm.is_a?(Category) - # Ignore the smaller categories, as they generally contain a sponsored - # channel, which brings a lot of noise on the trending page. - # See: https://github.com/iv-org/invidious/issues/2989 - next if (itm.contents.size < 24 && deduplicate) - - extracted.concat itm.contents.select(SearchItem) - else - extracted << itm - end - end - - # Deduplicate items before returning results - return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid + return {trending, plid} end diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr index b175c3b9..8a0f67e5 100644 --- a/src/invidious/user/captcha.cr +++ b/src/invidious/user/captcha.cr @@ -4,6 +4,8 @@ struct Invidious::User module Captcha extend self + private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com") + def generate_image(key) second = Random::Secure.rand(12) second_angle = second * 30 @@ -58,5 +60,19 @@ struct Invidious::User tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, } end + + def generate_text(key) + response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body) + response = JSON.parse(response) + + tokens = response["a"].as_a.map do |answer| + generate_response(answer.as_s, {":login"}, key, use_nonce: true) + end + + return { + question: response["q"].as_s, + tokens: tokens, + } + end end end diff --git a/src/invidious/user/exports.cr b/src/invidious/user/exports.cr deleted file mode 100644 index b52503c9..00000000 --- a/src/invidious/user/exports.cr +++ /dev/null @@ -1,35 +0,0 @@ -struct Invidious::User - module Export - extend self - - def to_invidious(user : User) - playlists = Invidious::Database::Playlists.select_like_iv(user.email) - - return JSON.build do |json| - json.object do - json.field "subscriptions", user.subscriptions - json.field "watch_history", user.watched - json.field "preferences", user.preferences - json.field "playlists" do - json.array do - playlists.each do |playlist| - json.object do - json.field "title", playlist.title - json.field "description", html_to_content(playlist.description_html) - json.field "privacy", playlist.privacy.to_s - json.field "videos" do - json.array do - Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: CONFIG.playlist_length_limit).each do |video_id| - json.string video_id - end - end - end - end - end - end - end - end - end - end - end # module -end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 007eb666..f8b9e4e4 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -6,7 +6,7 @@ struct Invidious::User # Parse a youtube CSV subscription file def parse_subscription_export_csv(csv_content : String) - rows = CSV.new(csv_content.strip('\n'), headers: true) + rows = CSV.new(csv_content, headers: true) subscriptions = Array(String).new # Counter to limit the amount of imports. @@ -30,60 +30,6 @@ struct Invidious::User return subscriptions end - def parse_playlist_export_csv(user : User, raw_input : String) - # Split the input into head and body content - raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true) - - # Create the playlist from the head content - csv_head = CSV.new(raw_head.strip('\n'), headers: true) - csv_head.next - title = csv_head[4] - description = csv_head[5] - visibility = csv_head[6] - - if visibility.compare("Public", case_insensitive: true) == 0 - privacy = PlaylistPrivacy::Public - else - privacy = PlaylistPrivacy::Private - end - - playlist = create_playlist(title, privacy, user) - Invidious::Database::Playlists.update_description(playlist.id, description) - - # Add each video to the playlist from the body content - csv_body = CSV.new(raw_body.strip('\n'), headers: true) - csv_body.each do |row| - video_id = row[0] - if playlist - next if !video_id - next if video_id == "Video Id" - - begin - video = get_video(video_id) - rescue ex - next - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist.id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - Invidious::Database::PlaylistVideos.insert(playlist_video) - Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) - end - end - - return playlist - end - # ------------------- # Invidious # ------------------- @@ -102,7 +48,7 @@ struct Invidious::User if data["watch_history"]? user.watched += data["watch_history"].as_a.map(&.as_s) - user.watched.reverse!.uniq!.reverse! + user.watched.uniq! Invidious::Database::Users.update_watch_history(user) end @@ -115,7 +61,7 @@ struct Invidious::User playlists.each do |item| title = item["title"]?.try &.as_s?.try &.delete("<>") description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state } + privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } next if !title next if !description @@ -124,16 +70,14 @@ struct Invidious::User playlist = create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) - item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| - if idx > CONFIG.playlist_length_limit - raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") - end + videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| + raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500 video_id = video_id.try &.as_s? next if !video_id begin - video = get_video(video_id, false) + video = get_video(video_id) rescue ex next end @@ -161,7 +105,7 @@ struct Invidious::User # Youtube # ------------------- - private def opml?(mimetype : String, extension : String) + private def is_opml?(mimetype : String, extension : String) opml_mimetypes = [ "application/xml", "text/xml", @@ -179,10 +123,10 @@ struct Invidious::User def from_youtube(user : User, body : String, filename : String, type : String) : Bool extension = filename.split(".").last - if opml?(type, extension) + if is_opml?(type, extension) subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| - channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0] + channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] end elsif extension == "json" || type == "application/json" subscriptions = JSON.parse(body) @@ -203,41 +147,6 @@ struct Invidious::User return true end - def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool - extension = filename.split(".").last - - if extension == "csv" || type == "text/csv" - playlist = parse_playlist_export_csv(user, body) - if playlist - return true - else - return false - end - else - return false - end - end - - def from_youtube_wh(user : User, body : String, filename : String, type : String) : Bool - extension = filename.split(".").last - - if extension == "json" || type == "application/json" - data = JSON.parse(body) - watched = data.as_a.compact_map do |item| - next unless url = item["titleUrl"]? - next unless match = url.as_s.match(/\?v=(?
+ <% videos.each_slice(4) do |slice| %>
+ <% slice.each do |item| %>
+ <%= rendered "components/item" %>
+ <% end %>
+ <% end %>
+
-<%= rendered "components/items_paginated" %>
+<% if query %>
+ <%- query_encoded = URI.encode_www_form(query.text, space_to_plus: true) -%>
+
+
+<% end %>
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 686de6bd..92f81ee4 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,59 +1,116 @@
-<%-
- ucid = channel.ucid
- author = HTML.escape(channel.author)
- channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
-
- relative_url =
- case selected_tab
- when .shorts? then "/channel/#{ucid}/shorts"
- when .streams? then "/channel/#{ucid}/streams"
- when .playlists? then "/channel/#{ucid}/playlists"
- when .channels? then "/channel/#{ucid}/channels"
- when .podcasts? then "/channel/#{ucid}/podcasts"
- when .releases? then "/channel/#{ucid}/releases"
- when .courses? then "/channel/#{ucid}/courses"
- else
- "/channel/#{ucid}"
- end
-
- youtube_url = "https://www.youtube.com#{relative_url}"
- redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-
- page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
- base_url: relative_url,
- ctoken: next_continuation,
- first_page: continuation.nil?,
- params: env.params.query,
- )
-%>
+<% ucid = channel.ucid %>
+<% author = HTML.escape(channel.author) %>
<% content_for "header" do %>
-<%- if selected_tab.videos? -%>
-
-
-
-
-
-
-
-
-
-
-
-
-<%- end -%>
-
-
-
-
+ <% if query.page > 1 %>
+
+ <%= translate(locale, "Previous page") %>
+
+ <% end %>
+
+
+
+ <% if videos.size >= 20 %>
+
+ <%= translate(locale, "Next page") %>
+
+ <% end %>
+
+
+ ">
+
+
+ ">
+
+
+
+<% end %>
+
+
+
++
.request_target %>)
+
+
+
+
+
+<%= channel.description_html %>
+
+ <% sub_count_text = number_to_short_text(channel.sub_count) %>
+ <%= rendered "components/subscribe_widget" %>
+
+
+
+
+ <%= translate(locale, "View channel on YouTube") %>
+
+
+
+ <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
+ "><%= translate(locale, "Switch Invidious Instance") %>
+ <% else %>
+ <%= translate(locale, "Switch Invidious Instance") %>
+ <% end %>
+
+ <% if !channel.auto_generated %>
+
+ <%= translate(locale, "Videos") %>
+
+ <% end %>
+
+ <% if channel.auto_generated %>
+ <%= translate(locale, "Playlists") %>
+ <% else %>
+ <%= translate(locale, "Playlists") %>
+ <% end %>
+
+
+ <% if channel.tabs.includes? "community" %>
+ <%= translate(locale, "Community") %>
+ <% end %>
+
+
+
+
+ <% sort_options.each do |sort| %>
+
+
+ <% if sort_by == sort %>
+ <%= translate(locale, sort) %>
+ <% else %>
+
+ <%= translate(locale, sort) %>
+
+ <% end %>
+
+ <% end %>
+
+<% items.each do |item| %>
+ <%= rendered "components/item" %>
+<% end %>
+
-<%= rendered "components/items_paginated" %>
+
+
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 132e636c..3bc29e55 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,21 +1,71 @@
-<%-
- ucid = channel.ucid
- author = HTML.escape(channel.author)
- channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
-
- relative_url = "/channel/#{ucid}/community"
- youtube_url = "https://www.youtube.com#{relative_url}"
- redirect_url = Invidious::Frontend::Misc.redirect_url(env)
-
- selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Posts
--%>
+<% ucid = channel.ucid %>
+<% author = HTML.escape(channel.author) %>
<% content_for "header" do %>
-
+ <% if page > 1 %>
+ &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <%= translate(locale, "Previous page") %>
+
+ <% end %>
+
+
+
+ <% if count == 60 %>
+ &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+ <%= translate(locale, "Next page") %>
+
+ <% end %>
+
+
+ ">
+
+
+ ">
+
+
+
+<% end %>
+
+
+
++
+
+
+
+
+<%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>
+
+ <% sub_count_text = number_to_short_text(channel.sub_count) %>
+ <%= rendered "components/subscribe_widget" %>
+
+
+
+
+ <%= translate(locale, "View channel on YouTube") %>
+
+
+
+ <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
+ "><%= translate(locale, "Switch Invidious Instance") %>
+ <% else %>
+ <%= translate(locale, "Switch Invidious Instance") %>
+ <% end %>
+
+ <% if !channel.auto_generated %>
+
+ <% end %>
+
+
+ <% if channel.tabs.includes? "community" %>
+ <%= translate(locale, "Community") %>
+ <% end %>
+
+ @@ -26,8 +76,8 @@
<%= error_message %>
-
<%= channel.description_html %>
<%= HTML.escape(item.author) %><% if !item.author_verified.nil? && item.author_verified %> <% end %>
+ +<%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>
+ <% if !item.auto_generated %><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
<% end %> +<%= item.description_html %>
+ <% when SearchPlaylist, InvidiousPlaylist %> + <% if item.id.starts_with? "RD" %> + <% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %> + <% else %> + <% url = "/playlist?list=#{item.id}" %> <% end %> + + <% if !env.get("preferences").as(Preferences).thin_mode %> +<%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
+<%= HTML.escape(item.title) %>
+ + +<%= HTML.escape(item.author) %><% if !item.is_a?(InvidiousPlaylist) && !item.author_verified.nil? && item.author_verified %> <% end %>
+ + <% when MixVideo %> + + <% if !env.get("preferences").as(Preferences).thin_mode %> +<%= recode_length_seconds(item.length_seconds) %>
+ <% end %> +<%= HTML.escape(item.title) %>
+ + +<%= HTML.escape(item.author) %>
+ + <% when PlaylistVideo %> + + <% if !env.get("preferences").as(Preferences).thin_mode %> +<%= translate(locale, "LIVE") %>
+ <% elsif item.length_seconds != 0 %> +<%= recode_length_seconds(item.length_seconds) %>
+ <% end %> +<%= HTML.escape(item.title) %>
+ +<%= HTML.escape(item.author) %> - <%- if author_verified %> <% end -%> -
+<%= HTML.escape(item.author) %>
<%= item.channel_handle %>
<% end %> -<%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>
- <% if !item.auto_generated && item.channel_handle.nil? %><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
<% end %> -<%= item.description_html %>
- <% when SearchHashtag %> - <% if !thin_mode %> - -<%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
- <%- end -%> -<%= translate_count(locale, "generic_channels_count", item.channel_count, NumberFormatting::Separator) %>
- <%- end -%> -<%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
-<%= HTML.escape(item.title) %>
+ <% endpoint_params = "?v=#{item.id}&list=#{item.plid}" %> + <%= rendered "components/video-context-buttons" %><%= HTML.escape(item.author) %> - <%- if author_verified %> <% end -%> -
- - <% else %> -<%= HTML.escape(item.author) %> - <%- if author_verified %> <% end -%> -
+ <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %> +<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
+ <% elsif Time.utc - item.published > 1.minute %> +<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% end %><%= translate_count(locale, "generic_views_count", item.views || 0, NumberFormatting::Short) %>
+<%=translate(locale, "timeline_parse_error_placeholder_heading")%>
-<%=translate(locale, "timeline_parse_error_placeholder_message")%>
-<%=translate(locale, "timeline_parse_error_show_technical_details")%>
- -<%= translate(locale, "LIVE") %>
- <%- elsif item.length_seconds != 0 -%> -<%= recode_length_seconds(item.length_seconds) %>
- <%- end -%> -<%= HTML.escape(item.title) %>
-<%= translate(locale, "LIVE") %>
+ <% elsif item.length_seconds != 0 %> +<%= recode_length_seconds(item.length_seconds) %>
+ <% end %> +<%= HTML.escape(item.title) %>
+<%= HTML.escape(item.author) %> - <%- if author_verified %> <% end -%> -
- - <% else %> -<%= HTML.escape(item.author) %> - <%- if author_verified %> <% end -%> -
- <% end %> -<%= HTML.escape(item.author) %><% if !item.is_a?(ChannelVideo) && !item.author_verified.nil? && item.author_verified %> <% end %>
+<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
- <% elsif item.responds_to?(:published) && (Time.utc - item.published) > 1.minute %> + <% elsif Time.utc - item.published > 1.minute %><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% end %>