-
-
- END_HTML
- end
-
- if !thin_mode
- author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).request_target}"
- else
- author_thumbnail = ""
- end
-
- author_name = HTML.escape(child["author"].as_s)
-
- html << <<-END_HTML
-
-
-
-
-
- END_HTML
- end
-
- if comments["continuation"]?
- html << <<-END_HTML
-
- 
-
-
-
- - - #{author_name} - -
#{child["contentHtml"]}
- END_HTML - - if child["attachment"]? - attachment = child["attachment"] - - case attachment["type"] - when "image" - attachment = attachment["imageThumbnails"][1] - - html << <<-END_HTML -
-
- END_HTML
- when "video"
- html << <<-END_HTML
-
- 
-
-
-
-
- END_HTML
- else nil # Ignore
- end
- end
-
- html << <<-END_HTML
- #{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}
- |
- END_HTML
-
- if comments["videoId"]?
- html << <<-END_HTML
- [YT]
- |
- END_HTML
- elsif comments["authorId"]?
- html << <<-END_HTML
- [YT]
- |
- END_HTML
- end
-
- html << <<-END_HTML
- #{number_with_separator(child["likeCount"])}
- END_HTML
-
- if child["creatorHeart"]?
- if !thin_mode
- creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).request_target}"
- else
- creator_thumbnail = ""
- end
-
- html << <<-END_HTML
-
-
-
-
- END_HTML
-
- if attachment["error"]?
- html << <<-END_HTML
-
- #{attachment["error"]}
- END_HTML - else - html << <<-END_HTML - - END_HTML - end - - html << <<-END_HTML -
- 
-
-
- END_HTML
- end
-
- html << <<-END_HTML
-
- #{replies_html}
-
-
-
-
-
-
- END_HTML
- end
- end
-end
-
-def template_reddit_comments(root, locale)
- String.build do |html|
- root.each do |child|
- if child.data.is_a?(RedditComment)
- child = child.data.as(RedditComment)
- body_html = HTML.unescape(child.body_html)
-
- replies_html = ""
- if child.replies.is_a?(RedditThing)
- replies = child.replies.as(RedditThing)
- replies_html = template_reddit_comments(replies.data.as(RedditListing).children, locale)
- end
-
- if child.depth > 0
- html << <<-END_HTML
-
-
-
-
- END_HTML
- else
- html << <<-END_HTML
-
") } || "" -end - -def content_to_comment_html(content) - comment_html = content.map do |run| - text = HTML.escape(run["text"].as_s) - - if run["bold"]? - text = "#{text}" - end - - if run["italics"]? - text = "#{text}" - end - - if run["navigationEndpoint"]? - if url = run["navigationEndpoint"]["urlEndpoint"]?.try &.["url"].as_s - url = URI.parse(url) - - if url.host == "youtu.be" - url = "/watch?v=#{url.request_target.lstrip('/')}" - elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") - if url.path == "/redirect" - # Sometimes, links can be corrupted (why?) so make sure to fallback - # nicely. See https://github.com/iv-org/invidious/issues/2682 - url = HTTP::Params.parse(url.query.not_nil!)["q"]? || "" - else - url = url.request_target - end - end - - text = %(#{text}) - elsif watch_endpoint = run["navigationEndpoint"]["watchEndpoint"]? - length_seconds = watch_endpoint["startTimeSeconds"]? - video_id = watch_endpoint["videoId"].as_s - - if length_seconds && length_seconds.as_i > 0 - text = %(#{text}) - else - text = %(#{text}) - end - elsif url = run.dig?("navigationEndpoint", "commandMetadata", "webCommandMetadata", "url").try &.as_s - text = %(#{text}) - end - end - - text - end.join("").delete('\ufeff') - - return comment_html -end - -def produce_comment_continuation(video_id, cursor = "", sort_by = "top") - object = { - "2:embedded" => { - "2:string" => video_id, - "25:varint" => 0_i64, - "28:varint" => 1_i64, - "36:embedded" => { - "5:varint" => -1_i64, - "8:varint" => 0_i64, - }, - "40:embedded" => { - "1:varint" => 4_i64, - "3:string" => "https://www.youtube.com", - "4:string" => "", - }, - }, - "3:varint" => 6_i64, - "6:embedded" => { - "1:string" => cursor, - "4:embedded" => { - "4:string" => video_id, - "6:varint" => 0_i64, - }, - "5:varint" => 20_i64, - }, - } - - case sort_by - when "top" - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 - when "new", "newest" - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64 - else # top - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 - end - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr new file mode 100644 index 00000000..1f55bfe6 --- /dev/null +++ b/src/invidious/comments/content.cr @@ -0,0 +1,89 @@ +def text_to_parsed_content(text : String) : JSON::Any + nodes = [] of JSON::Any + # For each line convert line to array of nodes + text.split('\n').each do |line| + # In first case line is just a simple node before + # check patterns inside line + # { 'text': line } + current_nodes = [] of JSON::Any + initial_node = {"text" => line} + current_nodes << (JSON.parse(initial_node.to_json)) + + # For each match with url pattern, get last node and preserve + # last node before create new node with url information + # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } + line.scan(/https?:\/\/[^ ]*/).each do |url_match| + # Retrieve last node and update node without match + last_node = current_nodes[-1].as_h + splitted_last_node = last_node["text"].as_s.split(url_match[0]) + last_node["text"] = JSON.parse(splitted_last_node[0].to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) + # Create new node with match and navigation infos + current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}} + current_nodes << (JSON.parse(current_node.to_json)) + # If text remain after match create new simple node with text after match + after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""} + current_nodes << (JSON.parse(after_node.to_json)) + end + + # After processing of matches inside line + # Add \n at end of last node for preserve carriage return + last_node = current_nodes[-1].as_h + last_node["text"] = JSON.parse("#{last_node["text"]}\n".to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) + + # Finally add final nodes to nodes returned + current_nodes.each do |node| + nodes << (node) + end + end + return JSON.parse({"runs" => nodes}.to_json) +end + +def parse_content(content : JSON::Any, video_id : String? = "") : String + content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || + content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r, video_id).try &.to_s.gsub("\n", "
") } || "" +end + +def content_to_comment_html(content, video_id : String? = "") + html_array = content.map do |run| + # Sometimes, there is an empty element. + # See: https://github.com/iv-org/invidious/issues/3096 + next if run.as_h.empty? + + text = HTML.escape(run["text"].as_s) + + if navigation_endpoint = run.dig?("navigationEndpoint") + text = parse_link_endpoint(navigation_endpoint, text, video_id) + end + + text = "#{text}" if run["bold"]? + text = "#{text}" if run["strikethrough"]?
+ text = "#{text}" if run["italics"]?
+
+ # check for custom emojis
+ if run["emoji"]?
+ if run["emoji"]["isCustomEmoji"]?.try &.as_bool
+ if emoji_image = run.dig?("emoji", "image")
+ emoji_alt = emoji_image.dig?("accessibility", "accessibilityData", "label").try &.as_s || text
+ emoji_thumb = emoji_image["thumbnails"][0]
+ text = String.build do |str|
+ str << %(![) << emoji_alt <<]()
)
+ end
+ else
+ # Hide deleted channel emoji
+ text = ""
+ end
+ end
+ end
+
+ text
+ end
+
+ return html_array.join("").delete('\ufeff')
+end
diff --git a/src/invidious/comments/links_util.cr b/src/invidious/comments/links_util.cr
new file mode 100644
index 00000000..f89b86d3
--- /dev/null
+++ b/src/invidious/comments/links_util.cr
@@ -0,0 +1,76 @@
+module Invidious::Comments
+ extend self
+
+ def replace_links(html)
+ # Check if the document is empty
+ # Prevents edge-case bug with Reddit comments, see issue #3115
+ if html.nil? || html.empty?
+ return html
+ end
+
+ html = XML.parse_html(html)
+
+ html.xpath_nodes(%q(//a)).each do |anchor|
+ url = URI.parse(anchor["href"])
+
+ if url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") || url.host.not_nil!.ends_with?("youtu.be")
+ if url.host.try &.ends_with? "youtu.be"
+ url = "/watch?v=#{url.path.lstrip('/')}#{url.query_params}"
+ else
+ if url.path == "/redirect"
+ params = HTTP::Params.parse(url.query.not_nil!)
+ anchor["href"] = params["q"]?
+ else
+ anchor["href"] = url.request_target
+ end
+ end
+ elsif url.to_s == "#"
+ begin
+ length_seconds = decode_length_seconds(anchor.content)
+ rescue ex
+ length_seconds = decode_time(anchor.content)
+ end
+
+ if length_seconds > 0
+ anchor["href"] = "javascript:void(0)"
+ anchor["onclick"] = "player.currentTime(#{length_seconds})"
+ else
+ anchor["href"] = url.request_target
+ end
+ end
+ end
+
+ html = html.xpath_node(%q(//body)).not_nil!
+ if node = html.xpath_node(%q(./p))
+ html = node
+ end
+
+ return html.to_xml(options: XML::SaveOptions::NO_DECL)
+ end
+
+ def fill_links(html, scheme, host)
+ # Check if the document is empty
+ # Prevents edge-case bug with Reddit comments, see issue #3115
+ if html.nil? || html.empty?
+ return html
+ end
+
+ html = XML.parse_html(html)
+
+ html.xpath_nodes("//a").each do |match|
+ url = URI.parse(match["href"])
+ # Reddit links don't have host
+ if !url.host && !match["href"].starts_with?("javascript") && !url.to_s.ends_with? "#"
+ url.scheme = scheme
+ url.host = host
+ match["href"] = url
+ end
+ end
+
+ if host == "www.youtube.com"
+ html = html.xpath_node(%q(//body/p)).not_nil!
+ end
+
+ return html.to_xml(options: XML::SaveOptions::NO_DECL)
+ end
+end
diff --git a/src/invidious/comments/reddit.cr b/src/invidious/comments/reddit.cr
new file mode 100644
index 00000000..ba9c19f1
--- /dev/null
+++ b/src/invidious/comments/reddit.cr
@@ -0,0 +1,41 @@
+module Invidious::Comments
+ extend self
+
+ def fetch_reddit(id, sort_by = "confidence")
+ client = make_client(REDDIT_URL)
+ headers = HTTP::Headers{"User-Agent" => "web:invidious:v#{CURRENT_VERSION} (by github.com/iv-org/invidious)"}
+
+ # TODO: Use something like #479 for a static list of instances to use here
+ query = URI::Params.encode({q: "(url:3D#{id} OR url:#{id}) AND (site:invidio.us OR site:youtube.com OR site:youtu.be)"})
+ search_results = client.get("/search.json?#{query}", headers)
+
+ if search_results.status_code == 200
+ search_results = RedditThing.from_json(search_results.body)
+
+ # For videos that have more than one thread, choose the one with the highest score
+ threads = search_results.data.as(RedditListing).children
+ thread = threads.max_by?(&.data.as(RedditLink).score).try(&.data.as(RedditLink))
+ result = thread.try do |t|
+ body = client.get("/r/#{t.subreddit}/comments/#{t.id}.json?limit=100&sort=#{sort_by}", headers).body
+ Array(RedditThing).from_json(body)
+ end
+ result ||= [] of RedditThing
+ elsif search_results.status_code == 302
+ # Previously, if there was only one result then the API would redirect to that result.
+ # Now, it appears it will still return a listing so this section is likely unnecessary.
+
+ result = client.get(search_results.headers["Location"], headers).body
+ result = Array(RedditThing).from_json(result)
+
+ thread = result[0].data.as(RedditListing).children[0].data.as(RedditLink)
+ else
+ raise NotFoundException.new("Comments not found.")
+ end
+
+ client.close
+
+ comments = result[1]?.try(&.data.as(RedditListing).children)
+ comments ||= [] of RedditThing
+ return comments, thread
+ end
+end
diff --git a/src/invidious/comments/reddit_types.cr b/src/invidious/comments/reddit_types.cr
new file mode 100644
index 00000000..796a1183
--- /dev/null
+++ b/src/invidious/comments/reddit_types.cr
@@ -0,0 +1,57 @@
+class RedditThing
+ include JSON::Serializable
+
+ property kind : String
+ property data : RedditComment | RedditLink | RedditMore | RedditListing
+end
+
+class RedditComment
+ include JSON::Serializable
+
+ property author : String
+ property body_html : String
+ property replies : RedditThing | String
+ property score : Int32
+ property depth : Int32
+ property permalink : String
+
+ @[JSON::Field(converter: RedditComment::TimeConverter)]
+ property created_utc : Time
+
+ module TimeConverter
+ def self.from_json(value : JSON::PullParser) : Time
+ Time.unix(value.read_float.to_i)
+ end
+
+ def self.to_json(value : Time, json : JSON::Builder)
+ json.number(value.to_unix)
+ end
+ end
+end
+
+struct RedditLink
+ include JSON::Serializable
+
+ property author : String
+ property score : Int32
+ property subreddit : String
+ property num_comments : Int32
+ property id : String
+ property permalink : String
+ property title : String
+end
+
+struct RedditMore
+ include JSON::Serializable
+
+ property children : Array(String)
+ property count : Int32
+ property depth : Int32
+end
+
+class RedditListing
+ include JSON::Serializable
+
+ property children : Array(RedditThing)
+ property modhash : String
+end
diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr
new file mode 100644
index 00000000..0716fcde
--- /dev/null
+++ b/src/invidious/comments/youtube.cr
@@ -0,0 +1,365 @@
+module Invidious::Comments
+ extend self
+
+ def fetch_youtube(id, cursor, format, locale, thin_mode, region, sort_by = "top")
+ case cursor
+ when nil, ""
+ ctoken = Comments.produce_continuation(id, cursor: "", sort_by: sort_by)
+ when .starts_with? "ADSJ"
+ ctoken = Comments.produce_continuation(id, cursor: cursor, sort_by: sort_by)
+ else
+ ctoken = cursor
+ end
+
+ client_config = YoutubeAPI::ClientConfig.new(region: region)
+ response = YoutubeAPI.next(continuation: ctoken, client_config: client_config)
+ return parse_youtube(id, response, format, locale, thin_mode, sort_by)
+ end
+
+ def fetch_community_post_comments(ucid, post_id)
+ object = {
+ "2:string" => "community",
+ "25:embedded" => {
+ "22:string" => post_id,
+ },
+ "45:embedded" => {
+ "2:varint" => 1_i64,
+ "3:varint" => 1_i64,
+ },
+ "53:embedded" => {
+ "4:embedded" => {
+ "6:varint" => 0_i64,
+ "27:varint" => 1_i64,
+ "29:string" => post_id,
+ "30:string" => ucid,
+ },
+ "8:string" => "comments-section",
+ },
+ }
+
+ object_parsed = object.try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+
+ object2 = {
+ "80226972:embedded" => {
+ "2:string" => ucid,
+ "3:string" => object_parsed,
+ },
+ }
+
+ continuation = object2.try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+
+ initial_data = YoutubeAPI.browse(continuation: continuation)
+ return initial_data
+ end
+
+ def parse_youtube(id, response, format, locale, thin_mode, sort_by = "top", is_post = false)
+ contents = nil
+
+ if on_response_received_endpoints = response["onResponseReceivedEndpoints"]?
+ header = nil
+ on_response_received_endpoints.as_a.each do |item|
+ if item["reloadContinuationItemsCommand"]?
+ case item["reloadContinuationItemsCommand"]["slot"]
+ when "RELOAD_CONTINUATION_SLOT_HEADER"
+ header = item["reloadContinuationItemsCommand"]["continuationItems"][0]
+ when "RELOAD_CONTINUATION_SLOT_BODY"
+ # continuationItems is nil when video has no comments
+ contents = item["reloadContinuationItemsCommand"]["continuationItems"]?
+ end
+ elsif item["appendContinuationItemsAction"]?
+ contents = item["appendContinuationItemsAction"]["continuationItems"]
+ end
+ end
+ elsif response["continuationContents"]?
+ response = response["continuationContents"]
+ if response["commentRepliesContinuation"]?
+ body = response["commentRepliesContinuation"]
+ else
+ body = response["itemSectionContinuation"]
+ end
+ contents = body["contents"]?
+ header = body["header"]?
+ else
+ raise NotFoundException.new("Comments not found.")
+ end
+
+ if !contents
+ if format == "json"
+ return {"comments" => [] of String}.to_json
+ else
+ return {"contentHtml" => "", "commentCount" => 0}.to_json
+ end
+ end
+
+ continuation_item_renderer = nil
+ contents.as_a.reject! do |item|
+ if item["continuationItemRenderer"]?
+ continuation_item_renderer = item["continuationItemRenderer"]
+ true
+ end
+ end
+
+ mutations = response.dig?("frameworkUpdates", "entityBatchUpdate", "mutations").try &.as_a || [] of JSON::Any
+
+ response = JSON.build do |json|
+ json.object do
+ if header
+ count_text = header["commentsHeaderRenderer"]["countText"]
+ comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?)
+ .try &.as_s.gsub(/\D/, "").to_i? || 0
+ json.field "commentCount", comment_count
+ end
+
+ if is_post
+ json.field "postId", id
+ else
+ json.field "videoId", id
+ end
+
+ json.field "comments" do
+ json.array do
+ contents.as_a.each do |node|
+ json.object do
+ if node["commentThreadRenderer"]?
+ node = node["commentThreadRenderer"]
+ end
+
+ if node["replies"]?
+ node_replies = node["replies"]["commentRepliesRenderer"]
+ end
+
+ if cvm = node["commentViewModel"]?
+ # two commentViewModels for inital request
+ # one commentViewModel when getting a replies to a comment
+ cvm = cvm["commentViewModel"] if cvm["commentViewModel"]?
+
+ comment_key = cvm["commentKey"]
+ toolbar_key = cvm["toolbarStateKey"]
+ comment_mutation = mutations.find { |i| i.dig?("payload", "commentEntityPayload", "key") == comment_key }
+ toolbar_mutation = mutations.find { |i| i.dig?("entityKey") == toolbar_key }
+
+ if !comment_mutation.nil? && !toolbar_mutation.nil?
+ # todo parse styleRuns, commandRuns and attachmentRuns for comments
+ html_content = parse_description(comment_mutation.dig("payload", "commentEntityPayload", "properties", "content"), id)
+ comment_author = comment_mutation.dig("payload", "commentEntityPayload", "author")
+ json.field "authorId", comment_author["channelId"].as_s
+ json.field "authorUrl", "/channel/#{comment_author["channelId"].as_s}"
+ json.field "author", comment_author["displayName"].as_s
+ json.field "verified", comment_author["isVerified"].as_bool
+ json.field "authorThumbnails" do
+ json.array do
+ comment_mutation.dig?("payload", "commentEntityPayload", "avatar", "image", "sources").try &.as_a.each do |thumbnail|
+ json.object do
+ json.field "url", thumbnail["url"]
+ json.field "width", thumbnail["width"]
+ json.field "height", thumbnail["height"]
+ end
+ end
+ end
+ end
+
+ json.field "authorIsChannelOwner", comment_author["isCreator"].as_bool
+ json.field "isSponsor", (comment_author["sponsorBadgeUrl"]? != nil)
+
+ if sponsor_badge_url = comment_author["sponsorBadgeUrl"]?
+ # Sponsor icon thumbnails always have one object and there's only ever the url property in it
+ json.field "sponsorIconUrl", sponsor_badge_url
+ end
+
+ comment_toolbar = comment_mutation.dig("payload", "commentEntityPayload", "toolbar")
+ json.field "likeCount", short_text_to_number(comment_toolbar["likeCountNotliked"].as_s)
+ reply_count = short_text_to_number(comment_toolbar["replyCount"]?.try &.as_s || "0")
+
+ if heart_state = toolbar_mutation.dig?("payload", "engagementToolbarStateEntityPayload", "heartState")
+ if heart_state.as_s == "TOOLBAR_HEART_STATE_HEARTED"
+ json.field "creatorHeart" do
+ json.object do
+ json.field "creatorThumbnail", comment_toolbar["creatorThumbnailUrl"].as_s
+ json.field "creatorName", comment_toolbar["heartActiveTooltip"].as_s.sub("❤ by ", "")
+ end
+ end
+ end
+ end
+
+ published_text = comment_mutation.dig?("payload", "commentEntityPayload", "properties", "publishedTime").try &.as_s
+ end
+
+ json.field "isPinned", (cvm.dig?("pinnedText") != nil)
+ json.field "commentId", cvm["commentId"]
+ else
+ if node["comment"]?
+ node_comment = node["comment"]["commentRenderer"]
+ else
+ node_comment = node["commentRenderer"]
+ end
+ json.field "commentId", node_comment["commentId"]
+ html_content = node_comment["contentText"]?.try { |t| parse_content(t, id) }
+
+ json.field "verified", (node_comment["authorCommentBadge"]? != nil)
+
+ json.field "author", node_comment["authorText"]?.try &.["simpleText"]? || ""
+ json.field "authorThumbnails" do
+ json.array do
+ node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail|
+ json.object do
+ json.field "url", thumbnail["url"]
+ json.field "width", thumbnail["width"]
+ json.field "height", thumbnail["height"]
+ end
+ end
+ end
+ end
+
+ if comment_action_buttons_renderer = node_comment.dig?("actionButtons", "commentActionButtonsRenderer")
+ json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i
+ if comment_action_buttons_renderer["creatorHeart"]?
+ heart_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"]
+ json.field "creatorHeart" do
+ json.object do
+ json.field "creatorThumbnail", heart_data["thumbnails"][-1]["url"]
+ json.field "creatorName", heart_data["accessibility"]["accessibilityData"]["label"]
+ end
+ end
+ end
+ end
+
+ if node_comment["authorEndpoint"]?
+ json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"]
+ json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"]
+ else
+ json.field "authorId", ""
+ json.field "authorUrl", ""
+ end
+
+ json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"]
+ json.field "isPinned", (node_comment["pinnedCommentBadge"]? != nil)
+ published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s
+
+ json.field "isSponsor", (node_comment["sponsorCommentBadge"]? != nil)
+ if node_comment["sponsorCommentBadge"]?
+ # Sponsor icon thumbnails always have one object and there's only ever the url property in it
+ json.field "sponsorIconUrl", node_comment.dig("sponsorCommentBadge", "sponsorCommentBadgeRenderer", "customBadge", "thumbnails", 0, "url").to_s
+ end
+
+ reply_count = node_comment["replyCount"]?
+ end
+
+ content_html = html_content || ""
+ json.field "content", html_to_content(content_html)
+ json.field "contentHtml", content_html
+
+ if published_text != nil
+ published_text = published_text.to_s
+ if published_text.includes?(" (edited)")
+ json.field "isEdited", true
+ published = decode_date(published_text.rchop(" (edited)"))
+ else
+ json.field "isEdited", false
+ published = decode_date(published_text)
+ end
+
+ json.field "published", published.to_unix
+ json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
+ end
+
+ if node_replies && !response["commentRepliesContinuation"]?
+ if node_replies["continuations"]?
+ continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
+ elsif node_replies["contents"]?
+ continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s
+ end
+ continuation ||= ""
+
+ json.field "replies" do
+ json.object do
+ json.field "replyCount", reply_count || 1
+ json.field "continuation", continuation
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ if continuation_item_renderer
+ if continuation_item_renderer["continuationEndpoint"]?
+ continuation_endpoint = continuation_item_renderer["continuationEndpoint"]
+ elsif continuation_item_renderer["button"]?
+ continuation_endpoint = continuation_item_renderer["button"]["buttonRenderer"]["command"]
+ end
+ if continuation_endpoint
+ json.field "continuation", continuation_endpoint["continuationCommand"]["token"].as_s
+ end
+ end
+ end
+ end
+
+ if format == "html"
+ response = JSON.parse(response)
+ content_html = Frontend::Comments.template_youtube(response, locale, thin_mode)
+ response = JSON.build do |json|
+ json.object do
+ json.field "contentHtml", content_html
+
+ if response["commentCount"]?
+ json.field "commentCount", response["commentCount"]
+ else
+ json.field "commentCount", 0
+ end
+ end
+ end
+ end
+
+ return response
+ end
+
+ def produce_continuation(video_id, cursor = "", sort_by = "top")
+ object = {
+ "2:embedded" => {
+ "2:string" => video_id,
+ "25:varint" => 0_i64,
+ "28:varint" => 1_i64,
+ "36:embedded" => {
+ "5:varint" => -1_i64,
+ "8:varint" => 0_i64,
+ },
+ "40:embedded" => {
+ "1:varint" => 4_i64,
+ "3:string" => "https://www.youtube.com",
+ "4:string" => "",
+ },
+ },
+ "3:varint" => 6_i64,
+ "6:embedded" => {
+ "1:string" => cursor,
+ "4:embedded" => {
+ "4:string" => video_id,
+ "6:varint" => 0_i64,
+ },
+ "5:varint" => 20_i64,
+ },
+ }
+
+ case sort_by
+ when "top"
+ object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
+ when "new", "newest"
+ object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64
+ else # top
+ object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64
+ end
+
+ continuation = object.try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+
+ return continuation
+ end
+end
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 93c4c0f7..4d69854c 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -8,11 +8,19 @@ struct DBConfig
property dbname : String
end
+struct SocketBindingConfig
+ include YAML::Serializable
+
+ property path : String
+ property permissions : String
+end
+
struct ConfigPreferences
include YAML::Serializable
property annotations : Bool = false
property annotations_subscribed : Bool = false
+ property preload : Bool = true
property autoplay : Bool = false
property captions : Array(String) = ["", "", ""]
property comments : Array(String) = ["youtube", ""]
@@ -27,7 +35,7 @@ struct ConfigPreferences
property max_results : Int32 = 40
property notifications_only : Bool = false
property player_style : String = "invidious"
- property quality : String = "hd720"
+ property quality : String = "dash"
property quality_dash : String = "auto"
property default_home : String? = "Popular"
property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
@@ -48,15 +56,34 @@ struct ConfigPreferences
def to_tuple
{% begin %}
{
- {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
+ {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}}
}
{% end %}
end
end
+struct HTTPProxyConfig
+ include YAML::Serializable
+
+ property user : String
+ property password : String
+ property host : String
+ property port : Int32
+end
+
class Config
include YAML::Serializable
+ class CompanionConfig
+ include YAML::Serializable
+
+ @[YAML::Field(converter: Preferences::URIConverter)]
+ property private_url : URI = URI.parse("")
+
+ @[YAML::Field(converter: Preferences::URIConverter)]
+ property public_url : URI = URI.parse("")
+ end
+
# Number of threads to use for crawling videos from channels (for updating subscriptions)
property channel_threads : Int32 = 1
# Time interval between two executions of the job that crawls channel videos (subscriptions update).
@@ -68,20 +95,24 @@ class Config
property output : String = "STDOUT"
# Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
property log_level : LogLevel = LogLevel::Info
+ # Enables colors in logs. Useful for debugging purposes
+ property colorize_logs : Bool = false
# Database configuration with separate parameters (username, hostname, etc)
property db : DBConfig? = nil
# Database configuration using 12-Factor "Database URL" syntax
@[YAML::Field(converter: Preferences::URIConverter)]
property database_url : URI = URI.parse("")
- # Use polling to keep decryption function up to date
- property decrypt_polling : Bool = true
# Used for crawling channels: threads should check all videos uploaded by a channel
property full_refresh : Bool = false
+
+ # Jobs config structure. See jobs.cr and jobs/base_job.cr
+ property jobs = Invidious::Jobs::JobsConfig.new
+
# Used to tell Invidious it is behind a proxy, so links to resources should be https://
property https_only : Bool?
# HMAC signing key for CSRF tokens and verifying pubsub subscriptions
- property hmac_key : String?
+ property hmac_key : String = ""
# Domain to be used for links to resources on the site where an absolute URL is required
property domain : String?
# Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
@@ -106,6 +137,8 @@ class Config
property hsts : Bool? = true
# Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
property disable_proxy : Bool? | Array(String)? = false
+ # Enable the user notifications for all users
+ property enable_user_notifications : Bool = true
# URL to the modified source code to be easily AGPL compliant
# Will display in the footer, next to the main source code link
@@ -114,22 +147,41 @@ class Config
# Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
@[YAML::Field(converter: Preferences::FamilyConverter)]
property force_resolve : Socket::Family = Socket::Family::UNSPEC
+
+ # External signature solver server socket (either a path to a UNIX domain socket or ":")
+ property signature_server : String? = nil
+
# Port to listen for connections (overridden by command line argument)
property port : Int32 = 3000
# Host to bind (overridden by command line argument)
property host_binding : String = "0.0.0.0"
+ # Path and permissions to make Invidious listen on a UNIX socket instead of a TCP port
+ property socket_binding : SocketBindingConfig? = nil
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100
- # Use quic transport for youtube api
- property use_quic : Bool = false
+ # HTTP Proxy configuration
+ property http_proxy : HTTPProxyConfig? = nil
+
+ # Use Innertube's transcripts API instead of timedtext for closed captions
+ property use_innertube_for_captions : Bool = false
+
+ # visitor data ID for Google session
+ property visitor_data : String? = nil
+ # poToken for passing bot attestation
+ property po_token : String? = nil
+
+ # Invidious companion
+ property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig
+
+ # Invidious companion API key
+ property invidious_companion_key : String = ""
# Saved cookies in "name1=value1; name2=value2..." format
@[YAML::Field(converter: Preferences::StringToCookies)]
property cookies : HTTP::Cookies = HTTP::Cookies.new
- # Key for Anti-Captcha
- property captcha_key : String? = nil
- # API URL for Anti-Captcha
- property captcha_api_url : String = "https://api.anti-captcha.com"
+
+ # Playlist length limit
+ property playlist_length_limit : Int32 = 500
def disabled?(option)
case disabled = CONFIG.disable_proxy
@@ -157,20 +209,20 @@ class Config
config = Config.from_yaml(config_yaml)
# Update config from env vars (upcased and prefixed with "INVIDIOUS_")
+ #
+ # Also checks if any top-level config options are set to "CHANGE_ME!!"
+ # TODO: Support non-top-level config options such as the ones in DBConfig
{% for ivar in Config.instance_vars %}
{% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
if ENV.has_key?({{env_id}})
- # puts %(Config.{{ivar.id}} : Loading from env var {{env_id}})
env_value = ENV.fetch({{env_id}})
success = false
# Use YAML converter if specified
{% ann = ivar.annotation(::YAML::Field) %}
{% if ann && ann[:converter] %}
- puts %(Config.{{ivar.id}} : Parsing "#{env_value}" as {{ivar.type}} with {{ann[:converter]}} converter)
config.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0])
- puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}})
success = true
# Use regular YAML parser otherwise
@@ -181,9 +233,7 @@ class Config
{{ivar_types}}.each do |ivar_type|
if !success
begin
- # puts %(Config.{{ivar.id}} : Trying to parse "#{env_value}" as #{ivar_type})
config.{{ivar.id}} = ivar_type.from_yaml(env_value)
- puts %(Config.{{ivar.id}} : Set to #{config.{{ivar.id}}} (#{ivar_type}))
success = true
rescue
# nop
@@ -198,8 +248,42 @@ class Config
exit(1)
end
end
+
+ # Warn when any config attribute is set to "CHANGE_ME!!"
+ if config.{{ivar.id}} == "CHANGE_ME!!"
+ puts "Config: The value of '#{ {{ivar.stringify}} }' needs to be changed!!"
+ exit(1)
+ end
{% end %}
+ if config.invidious_companion.present?
+ # invidious_companion and signature_server can't work together
+ if config.signature_server
+ puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."
+ exit(1)
+ elsif config.invidious_companion_key.empty?
+ puts "Config: Please configure a key if you are using invidious companion."
+ exit(1)
+ elsif config.invidious_companion_key == "CHANGE_ME!!"
+ puts "Config: The value of 'invidious_companion_key' needs to be changed!!"
+ exit(1)
+ elsif config.invidious_companion_key.size != 16
+ puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 characters."
+ exit(1)
+ end
+ elsif config.signature_server
+ puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/companion-installation/")
+ else
+ puts("WARNING: Invidious companion is required to view and playback videos. For more information see https://docs.invidious.io/companion-installation/")
+ end
+
+ # HMAC_key is mandatory
+ # See: https://github.com/iv-org/invidious/issues/3854
+ if config.hmac_key.empty?
+ puts "Config: 'hmac_key' is required/can't be empty"
+ exit(1)
+ end
+
# Build database_url from db.* if it's not set directly
if config.database_url.to_s.empty?
if db = config.db
@@ -212,7 +296,25 @@ class Config
path: db.dbname,
)
else
- puts "Config : Either database_url or db.* is required"
+ puts "Config: Either database_url or db.* is required"
+ exit(1)
+ end
+ end
+
+ # Check if the socket configuration is valid
+ if sb = config.socket_binding
+ if sb.path.ends_with?("/") || File.directory?(sb.path)
+ puts "Config: The socket path " + sb.path + " must not be a directory!"
+ exit(1)
+ end
+ d = File.dirname(sb.path)
+ if !File.directory?(d)
+ puts "Config: Socket directory " + sb.path + " does not exist or is not a directory!"
+ exit(1)
+ end
+ p = sb.permissions.to_i?(base: 8)
+ if !p || p < 0 || p > 0o777
+ puts "Config: Socket permissions must be an octal between 0 and 777!"
exit(1)
end
end
diff --git a/src/invidious/database/migration.cr b/src/invidious/database/migration.cr
new file mode 100644
index 00000000..921d8f38
--- /dev/null
+++ b/src/invidious/database/migration.cr
@@ -0,0 +1,38 @@
+abstract class Invidious::Database::Migration
+ macro inherited
+ Migrator.migrations << self
+ end
+
+ @@version : Int64?
+
+ def self.version(version : Int32 | Int64)
+ @@version = version.to_i64
+ end
+
+ getter? completed = false
+
+ def initialize(@db : DB::Database)
+ end
+
+ abstract def up(conn : DB::Connection)
+
+ def migrate
+ # migrator already ignores completed migrations
+ # but this is an extra check to make sure a migration doesn't run twice
+ return if completed?
+
+ @db.transaction do |txn|
+ up(txn.connection)
+ track(txn.connection)
+ @completed = true
+ end
+ end
+
+ def version : Int64
+ @@version.not_nil!
+ end
+
+ private def track(conn : DB::Connection)
+ conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version)
+ end
+end
diff --git a/src/invidious/database/migrations/0001_create_channels_table.cr b/src/invidious/database/migrations/0001_create_channels_table.cr
new file mode 100644
index 00000000..a1362bcf
--- /dev/null
+++ b/src/invidious/database/migrations/0001_create_channels_table.cr
@@ -0,0 +1,30 @@
+module Invidious::Database::Migrations
+ class CreateChannelsTable < Migration
+ version 1
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS public.channels
+ (
+ id text NOT NULL,
+ author text,
+ updated timestamp with time zone,
+ deleted boolean,
+ subscribed timestamp with time zone,
+ CONSTRAINT channels_id_key UNIQUE (id)
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON TABLE public.channels TO current_user;
+ SQL
+
+ conn.exec <<-SQL
+ CREATE INDEX IF NOT EXISTS channels_id_idx
+ ON public.channels
+ USING btree
+ (id COLLATE pg_catalog."default");
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0002_create_videos_table.cr b/src/invidious/database/migrations/0002_create_videos_table.cr
new file mode 100644
index 00000000..c2ac84f8
--- /dev/null
+++ b/src/invidious/database/migrations/0002_create_videos_table.cr
@@ -0,0 +1,28 @@
+module Invidious::Database::Migrations
+ class CreateVideosTable < Migration
+ version 2
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ CREATE UNLOGGED TABLE IF NOT EXISTS public.videos
+ (
+ id text NOT NULL,
+ info text,
+ updated timestamp with time zone,
+ CONSTRAINT videos_pkey PRIMARY KEY (id)
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON TABLE public.videos TO current_user;
+ SQL
+
+ conn.exec <<-SQL
+ CREATE UNIQUE INDEX IF NOT EXISTS id_idx
+ ON public.videos
+ USING btree
+ (id COLLATE pg_catalog."default");
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0003_create_channel_videos_table.cr b/src/invidious/database/migrations/0003_create_channel_videos_table.cr
new file mode 100644
index 00000000..c9b62e4c
--- /dev/null
+++ b/src/invidious/database/migrations/0003_create_channel_videos_table.cr
@@ -0,0 +1,35 @@
+module Invidious::Database::Migrations
+ class CreateChannelVideosTable < Migration
+ version 3
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS public.channel_videos
+ (
+ id text NOT NULL,
+ title text,
+ published timestamp with time zone,
+ updated timestamp with time zone,
+ ucid text,
+ author text,
+ length_seconds integer,
+ live_now boolean,
+ premiere_timestamp timestamp with time zone,
+ views bigint,
+ CONSTRAINT channel_videos_id_key UNIQUE (id)
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON TABLE public.channel_videos TO current_user;
+ SQL
+
+ conn.exec <<-SQL
+ CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx
+ ON public.channel_videos
+ USING btree
+ (ucid COLLATE pg_catalog."default");
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0004_create_users_table.cr b/src/invidious/database/migrations/0004_create_users_table.cr
new file mode 100644
index 00000000..a13ba15f
--- /dev/null
+++ b/src/invidious/database/migrations/0004_create_users_table.cr
@@ -0,0 +1,34 @@
+module Invidious::Database::Migrations
+ class CreateUsersTable < Migration
+ version 4
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS public.users
+ (
+ updated timestamp with time zone,
+ notifications text[],
+ subscriptions text[],
+ email text NOT NULL,
+ preferences text,
+ password text,
+ token text,
+ watched text[],
+ feed_needs_update boolean,
+ CONSTRAINT users_email_key UNIQUE (email)
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON TABLE public.users TO current_user;
+ SQL
+
+ conn.exec <<-SQL
+ CREATE UNIQUE INDEX IF NOT EXISTS email_unique_idx
+ ON public.users
+ USING btree
+ (lower(email) COLLATE pg_catalog."default");
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0005_create_session_ids_table.cr b/src/invidious/database/migrations/0005_create_session_ids_table.cr
new file mode 100644
index 00000000..13c2228d
--- /dev/null
+++ b/src/invidious/database/migrations/0005_create_session_ids_table.cr
@@ -0,0 +1,28 @@
+module Invidious::Database::Migrations
+ class CreateSessionIdsTable < Migration
+ version 5
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS public.session_ids
+ (
+ id text NOT NULL,
+ email text,
+ issued timestamp with time zone,
+ CONSTRAINT session_ids_pkey PRIMARY KEY (id)
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON TABLE public.session_ids TO current_user;
+ SQL
+
+ conn.exec <<-SQL
+ CREATE INDEX IF NOT EXISTS session_ids_id_idx
+ ON public.session_ids
+ USING btree
+ (id COLLATE pg_catalog."default");
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0006_create_nonces_table.cr b/src/invidious/database/migrations/0006_create_nonces_table.cr
new file mode 100644
index 00000000..cf1229e1
--- /dev/null
+++ b/src/invidious/database/migrations/0006_create_nonces_table.cr
@@ -0,0 +1,27 @@
+module Invidious::Database::Migrations
+ class CreateNoncesTable < Migration
+ version 6
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS public.nonces
+ (
+ nonce text,
+ expire timestamp with time zone,
+ CONSTRAINT nonces_id_key UNIQUE (nonce)
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON TABLE public.nonces TO current_user;
+ SQL
+
+ conn.exec <<-SQL
+ CREATE INDEX IF NOT EXISTS nonces_nonce_idx
+ ON public.nonces
+ USING btree
+ (nonce COLLATE pg_catalog."default");
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0007_create_annotations_table.cr b/src/invidious/database/migrations/0007_create_annotations_table.cr
new file mode 100644
index 00000000..dcecbc3b
--- /dev/null
+++ b/src/invidious/database/migrations/0007_create_annotations_table.cr
@@ -0,0 +1,20 @@
+module Invidious::Database::Migrations
+ class CreateAnnotationsTable < Migration
+ version 7
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS public.annotations
+ (
+ id text NOT NULL,
+ annotations xml,
+ CONSTRAINT annotations_id_key UNIQUE (id)
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON TABLE public.annotations TO current_user;
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0008_create_playlists_table.cr b/src/invidious/database/migrations/0008_create_playlists_table.cr
new file mode 100644
index 00000000..6aa16e1a
--- /dev/null
+++ b/src/invidious/database/migrations/0008_create_playlists_table.cr
@@ -0,0 +1,50 @@
+module Invidious::Database::Migrations
+ class CreatePlaylistsTable < Migration
+ version 8
+
+ def up(conn : DB::Connection)
+ if !privacy_type_exists?(conn)
+ conn.exec <<-SQL
+ CREATE TYPE public.privacy AS ENUM
+ (
+ 'Public',
+ 'Unlisted',
+ 'Private'
+ );
+ SQL
+ end
+
+ conn.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS public.playlists
+ (
+ title text,
+ id text primary key,
+ author text,
+ description text,
+ video_count integer,
+ created timestamptz,
+ updated timestamptz,
+ privacy privacy,
+ index int8[]
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON public.playlists TO current_user;
+ SQL
+ end
+
+ private def privacy_type_exists?(conn : DB::Connection) : Bool
+ request = <<-SQL
+ SELECT 1 AS one
+ FROM pg_type
+ INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace
+ WHERE pg_namespace.nspname = 'public'
+ AND pg_type.typname = 'privacy'
+ LIMIT 1;
+ SQL
+
+ !conn.query_one?(request, as: Int32).nil?
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0009_create_playlist_videos_table.cr b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr
new file mode 100644
index 00000000..84938b9b
--- /dev/null
+++ b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr
@@ -0,0 +1,27 @@
+module Invidious::Database::Migrations
+ class CreatePlaylistVideosTable < Migration
+ version 9
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS public.playlist_videos
+ (
+ title text,
+ id text,
+ author text,
+ ucid text,
+ length_seconds integer,
+ published timestamptz,
+ plid text references playlists(id),
+ index int8,
+ live_now boolean,
+ PRIMARY KEY (index,plid)
+ );
+ SQL
+
+ conn.exec <<-SQL
+ GRANT ALL ON TABLE public.playlist_videos TO current_user;
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrations/0010_make_videos_unlogged.cr b/src/invidious/database/migrations/0010_make_videos_unlogged.cr
new file mode 100644
index 00000000..f5d19683
--- /dev/null
+++ b/src/invidious/database/migrations/0010_make_videos_unlogged.cr
@@ -0,0 +1,11 @@
+module Invidious::Database::Migrations
+ class MakeVideosUnlogged < Migration
+ version 10
+
+ def up(conn : DB::Connection)
+ conn.exec <<-SQL
+ ALTER TABLE public.videos SET UNLOGGED;
+ SQL
+ end
+ end
+end
diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr
new file mode 100644
index 00000000..660c3203
--- /dev/null
+++ b/src/invidious/database/migrator.cr
@@ -0,0 +1,49 @@
+class Invidious::Database::Migrator
+ MIGRATIONS_TABLE = "public.invidious_migrations"
+
+ class_getter migrations = [] of Invidious::Database::Migration.class
+
+ def initialize(@db : DB::Database)
+ end
+
+ def migrate
+ versions = load_versions
+
+ ran_migration = false
+ load_migrations.sort_by(&.version)
+ .each do |migration|
+ next if versions.includes?(migration.version)
+
+ puts "Running migration: #{migration.class.name}"
+ migration.migrate
+ ran_migration = true
+ end
+
+ puts "No migrations to run." unless ran_migration
+ end
+
+ def pending_migrations? : Bool
+ versions = load_versions
+
+ load_migrations.sort_by(&.version)
+ .any? { |migration| !versions.includes?(migration.version) }
+ end
+
+ private def load_migrations : Array(Invidious::Database::Migration)
+ self.class.migrations.map(&.new(@db))
+ end
+
+ private def load_versions : Array(Int64)
+ create_migrations_table
+ @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64)
+ end
+
+ private def create_migrations_table
+ @db.exec <<-SQL
+ CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} (
+ id bigserial PRIMARY KEY,
+ version bigint NOT NULL
+ )
+ SQL
+ end
+end
diff --git a/src/invidious/database/nonces.cr b/src/invidious/database/nonces.cr
index 469fcbd8..b87c81ec 100644
--- a/src/invidious/database/nonces.cr
+++ b/src/invidious/database/nonces.cr
@@ -4,7 +4,7 @@ module Invidious::Database::Nonces
extend self
# -------------------
- # Insert
+ # Insert / Delete
# -------------------
def insert(nonce : String, expire : Time)
@@ -17,6 +17,15 @@ module Invidious::Database::Nonces
PG_DB.exec(request, nonce, expire)
end
+ def delete_expired
+ request = <<-SQL
+ DELETE FROM nonces *
+ WHERE expire < now()
+ SQL
+
+ PG_DB.exec(request)
+ end
+
# -------------------
# Update
# -------------------
diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr
index c6754a1e..6dbcaa05 100644
--- a/src/invidious/database/playlists.cr
+++ b/src/invidious/database/playlists.cr
@@ -91,7 +91,7 @@ module Invidious::Database::Playlists
end
# -------------------
- # Salect
+ # Select
# -------------------
def select(*, id : String) : InvidiousPlaylist?
@@ -113,7 +113,7 @@ module Invidious::Database::Playlists
end
# -------------------
- # Salect (filtered)
+ # Select (filtered)
# -------------------
def select_like_iv(email : String) : Array(InvidiousPlaylist)
@@ -140,6 +140,7 @@ module Invidious::Database::Playlists
request = <<-SQL
SELECT id,title FROM playlists
WHERE author = $1 AND id LIKE 'IV%'
+ ORDER BY title
SQL
PG_DB.query_all(request, email, as: {String, String})
@@ -212,7 +213,7 @@ module Invidious::Database::PlaylistVideos
end
# -------------------
- # Salect
+ # Select
# -------------------
def select(plid : String, index : VideoIndex, offset, limit = 100) : Array(PlaylistVideo)
diff --git a/src/invidious/database/statistics.cr b/src/invidious/database/statistics.cr
index 1df549e2..9e4963fd 100644
--- a/src/invidious/database/statistics.cr
+++ b/src/invidious/database/statistics.cr
@@ -15,7 +15,7 @@ module Invidious::Database::Statistics
PG_DB.query_one(request, as: Int64)
end
- def count_users_active_1m : Int64
+ def count_users_active_6m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '6 months'
@@ -24,7 +24,7 @@ module Invidious::Database::Statistics
PG_DB.query_one(request, as: Int64)
end
- def count_users_active_6m : Int64
+ def count_users_active_1m : Int64
request = <<-SQL
SELECT count(*) FROM users
WHERE CURRENT_TIMESTAMP - updated < '1 month'
diff --git a/src/invidious/database/users.cr b/src/invidious/database/users.cr
index f62b43ea..4a3056ea 100644
--- a/src/invidious/database/users.cr
+++ b/src/invidious/database/users.cr
@@ -52,7 +52,7 @@ module Invidious::Database::Users
def mark_watched(user : User, vid : String)
request = <<-SQL
UPDATE users
- SET watched = array_append(watched, $1)
+ SET watched = array_append(array_remove(watched, $1), $1)
WHERE email = $2
SQL
@@ -119,15 +119,15 @@ module Invidious::Database::Users
# Update (notifs)
# -------------------
- def add_notification(video : ChannelVideo)
+ def add_multiple_notifications(channel_id : String, video_ids : Array(String))
request = <<-SQL
UPDATE users
- SET notifications = array_append(notifications, $1),
+ SET notifications = array_cat(notifications, $1),
feed_needs_update = true
WHERE $2 = ANY(subscriptions)
SQL
- PG_DB.exec(request, video.id, video.ucid)
+ PG_DB.exec(request, video_ids, channel_id)
end
def remove_notification(user : User, vid : String)
@@ -154,6 +154,16 @@ module Invidious::Database::Users
# Update (misc)
# -------------------
+ def feed_needs_update(channel_id : String)
+ request = <<-SQL
+ UPDATE users
+ SET feed_needs_update = true
+ WHERE $1 = ANY(subscriptions)
+ SQL
+
+ PG_DB.exec(request, channel_id)
+ end
+
def update_preferences(user : User)
request = <<-SQL
UPDATE users
diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr
index e1fa01c3..695f5b33 100644
--- a/src/invidious/database/videos.cr
+++ b/src/invidious/database/videos.cr
@@ -22,6 +22,15 @@ module Invidious::Database::Videos
PG_DB.exec(request, id)
end
+ def delete_expired
+ request = <<-SQL
+ DELETE FROM videos *
+ WHERE updated < (now() - interval '6 hours')
+ SQL
+
+ PG_DB.exec(request)
+ end
+
def update(video : Video)
request = <<-SQL
UPDATE videos
diff --git a/src/invidious/exceptions.cr b/src/invidious/exceptions.cr
index 490d98cd..690db907 100644
--- a/src/invidious/exceptions.cr
+++ b/src/invidious/exceptions.cr
@@ -1,3 +1,19 @@
+# InfoExceptions are for displaying information to the user.
+#
+# An InfoException might or might not indicate that something went wrong.
+# Historically Invidious didn't differentiate between these two options, so to
+# maintain previous functionality InfoExceptions do not print backtraces.
+class InfoException < Exception
+end
+
+# Exception used to hold the bogus UCID during a channel search.
+class ChannelSearchException < InfoException
+ getter channel : String
+
+ def initialize(@channel)
+ end
+end
+
# Exception used to hold the name of the missing item
# Should be used in all parsing functions
class BrokenTubeException < Exception
@@ -10,3 +26,15 @@ class BrokenTubeException < Exception
return "Missing JSON element \"#{@element}\""
end
end
+
+# Exception threw when an element is not found.
+class NotFoundException < InfoException
+end
+
+class VideoNotAvailableException < Exception
+end
+
+# Exception used to indicate that the JSON response from YT is missing
+# some important informations, and that the query should be sent again.
+class RetryOnceException < Exception
+end
diff --git a/src/invidious/frontend/channel_page.cr b/src/invidious/frontend/channel_page.cr
new file mode 100644
index 00000000..4fe21b96
--- /dev/null
+++ b/src/invidious/frontend/channel_page.cr
@@ -0,0 +1,47 @@
+module Invidious::Frontend::ChannelPage
+ extend self
+
+ enum TabsAvailable
+ Videos
+ Shorts
+ Streams
+ Podcasts
+ Releases
+ Courses
+ Playlists
+ Posts
+ Channels
+ end
+
+ def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
+ return String.build(1500) do |str|
+ base_url = "/channel/#{channel.ucid}"
+
+ TabsAvailable.each do |tab|
+ # Ignore playlists, as it is not supported for auto-generated channels yet
+ next if (tab.playlists? && channel.auto_generated)
+
+ tab_name = tab.to_s.downcase
+
+ if channel.tabs.includes? tab_name
+ str << %(
-
- END_HTML
- end
- end
- end
-end
-
-def replace_links(html)
- html = XML.parse_html(html)
-
- html.xpath_nodes(%q(//a)).each do |anchor|
- url = URI.parse(anchor["href"])
-
- if url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") || url.host.not_nil!.ends_with?("youtu.be")
- if url.host.try &.ends_with? "youtu.be"
- url = "/watch?v=#{url.path.lstrip('/')}#{url.query_params}"
- else
- if url.path == "/redirect"
- params = HTTP::Params.parse(url.query.not_nil!)
- anchor["href"] = params["q"]?
- else
- anchor["href"] = url.request_target
- end
- end
- elsif url.to_s == "#"
- begin
- length_seconds = decode_length_seconds(anchor.content)
- rescue ex
- length_seconds = decode_time(anchor.content)
- end
-
- if length_seconds > 0
- anchor["href"] = "javascript:void(0)"
- anchor["onclick"] = "player.currentTime(#{length_seconds})"
- else
- anchor["href"] = url.request_target
- end
- end
- end
-
- html = html.xpath_node(%q(//body)).not_nil!
- if node = html.xpath_node(%q(./p))
- html = node
- end
-
- return html.to_xml(options: XML::SaveOptions::NO_DECL)
-end
-
-def fill_links(html, scheme, host)
- html = XML.parse_html(html)
-
- html.xpath_nodes("//a").each do |match|
- url = URI.parse(match["href"])
- # Reddit links don't have host
- if !url.host && !match["href"].starts_with?("javascript") && !url.to_s.ends_with? "#"
- url.scheme = scheme
- url.host = host
- match["href"] = url
- end
- end
-
- if host == "www.youtube.com"
- html = html.xpath_node(%q(//body/p)).not_nil!
- end
-
- return html.to_xml(options: XML::SaveOptions::NO_DECL)
-end
-
-def parse_content(content : JSON::Any) : String
- content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s ||
- content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r).try &.to_s.gsub("\n", "
- END_HTML
- end
-
- html << <<-END_HTML
-
- - [ - ] - #{child.author} - #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} - #{translate(locale, "`x` ago", recode_date(child.created_utc, locale))} - #{translate(locale, "permalink")} -
-
- #{body_html}
- #{replies_html}
-
- ") } || "" -end - -def content_to_comment_html(content) - comment_html = content.map do |run| - text = HTML.escape(run["text"].as_s) - - if run["bold"]? - text = "#{text}" - end - - if run["italics"]? - text = "#{text}" - end - - if run["navigationEndpoint"]? - if url = run["navigationEndpoint"]["urlEndpoint"]?.try &.["url"].as_s - url = URI.parse(url) - - if url.host == "youtu.be" - url = "/watch?v=#{url.request_target.lstrip('/')}" - elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com") - if url.path == "/redirect" - # Sometimes, links can be corrupted (why?) so make sure to fallback - # nicely. See https://github.com/iv-org/invidious/issues/2682 - url = HTTP::Params.parse(url.query.not_nil!)["q"]? || "" - else - url = url.request_target - end - end - - text = %(#{text}) - elsif watch_endpoint = run["navigationEndpoint"]["watchEndpoint"]? - length_seconds = watch_endpoint["startTimeSeconds"]? - video_id = watch_endpoint["videoId"].as_s - - if length_seconds && length_seconds.as_i > 0 - text = %(#{text}) - else - text = %(#{text}) - end - elsif url = run.dig?("navigationEndpoint", "commandMetadata", "webCommandMetadata", "url").try &.as_s - text = %(#{text}) - end - end - - text - end.join("").delete('\ufeff') - - return comment_html -end - -def produce_comment_continuation(video_id, cursor = "", sort_by = "top") - object = { - "2:embedded" => { - "2:string" => video_id, - "25:varint" => 0_i64, - "28:varint" => 1_i64, - "36:embedded" => { - "5:varint" => -1_i64, - "8:varint" => 0_i64, - }, - "40:embedded" => { - "1:varint" => 4_i64, - "3:string" => "https://www.youtube.com", - "4:string" => "", - }, - }, - "3:varint" => 6_i64, - "6:embedded" => { - "1:string" => cursor, - "4:embedded" => { - "4:string" => video_id, - "6:varint" => 0_i64, - }, - "5:varint" => 20_i64, - }, - } - - case sort_by - when "top" - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 - when "new", "newest" - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 1_i64 - else # top - object["6:embedded"].as(Hash)["4:embedded"].as(Hash)["6:varint"] = 0_i64 - end - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr new file mode 100644 index 00000000..1f55bfe6 --- /dev/null +++ b/src/invidious/comments/content.cr @@ -0,0 +1,89 @@ +def text_to_parsed_content(text : String) : JSON::Any + nodes = [] of JSON::Any + # For each line convert line to array of nodes + text.split('\n').each do |line| + # In first case line is just a simple node before + # check patterns inside line + # { 'text': line } + current_nodes = [] of JSON::Any + initial_node = {"text" => line} + current_nodes << (JSON.parse(initial_node.to_json)) + + # For each match with url pattern, get last node and preserve + # last node before create new node with url information + # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } + line.scan(/https?:\/\/[^ ]*/).each do |url_match| + # Retrieve last node and update node without match + last_node = current_nodes[-1].as_h + splitted_last_node = last_node["text"].as_s.split(url_match[0]) + last_node["text"] = JSON.parse(splitted_last_node[0].to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) + # Create new node with match and navigation infos + current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}} + current_nodes << (JSON.parse(current_node.to_json)) + # If text remain after match create new simple node with text after match + after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""} + current_nodes << (JSON.parse(after_node.to_json)) + end + + # After processing of matches inside line + # Add \n at end of last node for preserve carriage return + last_node = current_nodes[-1].as_h + last_node["text"] = JSON.parse("#{last_node["text"]}\n".to_json) + current_nodes[-1] = JSON.parse(last_node.to_json) + + # Finally add final nodes to nodes returned + current_nodes.each do |node| + nodes << (node) + end + end + return JSON.parse({"runs" => nodes}.to_json) +end + +def parse_content(content : JSON::Any, video_id : String? = "") : String + content["simpleText"]?.try &.as_s.rchop('\ufeff').try { |b| HTML.escape(b) }.to_s || + content["runs"]?.try &.as_a.try { |r| content_to_comment_html(r, video_id).try &.to_s.gsub("\n", "
") } || "" +end + +def content_to_comment_html(content, video_id : String? = "") + html_array = content.map do |run| + # Sometimes, there is an empty element. + # See: https://github.com/iv-org/invidious/issues/3096 + next if run.as_h.empty? + + text = HTML.escape(run["text"].as_s) + + if navigation_endpoint = run.dig?("navigationEndpoint") + text = parse_link_endpoint(navigation_endpoint, text, video_id) + end + + text = "#{text}" if run["bold"]? + text = "
\n)
+
+ if tab == selected_tab
+ str << "\t"
+ str << translate(locale, "channel_tab_#{tab_name}_label")
+ str << "\n"
+ else
+ # Video tab doesn't have the last path component
+ url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
+
+ str << %(\t)
+ str << translate(locale, "channel_tab_#{tab_name}_label")
+ str << "\n"
+ end
+
+ str << "
"
+ end
+ end
+ end
+ end
+end
diff --git a/src/invidious/frontend/comments_reddit.cr b/src/invidious/frontend/comments_reddit.cr
new file mode 100644
index 00000000..4dda683e
--- /dev/null
+++ b/src/invidious/frontend/comments_reddit.cr
@@ -0,0 +1,50 @@
+module Invidious::Frontend::Comments
+ extend self
+
+ def template_reddit(root, locale)
+ String.build do |html|
+ root.each do |child|
+ if child.data.is_a?(RedditComment)
+ child = child.data.as(RedditComment)
+ body_html = HTML.unescape(child.body_html)
+
+ replies_html = ""
+ if child.replies.is_a?(RedditThing)
+ replies = child.replies.as(RedditThing)
+ replies_html = self.template_reddit(replies.data.as(RedditListing).children, locale)
+ end
+
+ if child.depth > 0
+ html << <<-END_HTML
+
+
+
+
+ END_HTML
+ else
+ html << <<-END_HTML
+ ![]()
)
+ end
+ end
+ html << <<-END_HTML
+
- #{go_to_youtube}
+ #{go_to_youtube}
END_HTML
diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr
index d140a858..13ea9fe9 100644
--- a/src/invidious/helpers/handlers.cr
+++ b/src/invidious/helpers/handlers.cr
@@ -27,6 +27,7 @@ class Kemal::RouteHandler
# Processes the route if it's a match. Otherwise renders 404.
private def process_request(context)
raise Kemal::Exceptions::RouteNotFound.new(context) unless context.route_found?
+ return if context.response.closed?
content = context.route.handler.call(context)
if !Kemal.config.error_handlers.empty? && Kemal.config.error_handlers.has_key?(context.response.status_code) && exclude_match?(context)
@@ -97,7 +98,7 @@ class AuthHandler < Kemal::Handler
if token = env.request.headers["Authorization"]?
token = JSON.parse(URI.decode_www_form(token.lchop("Bearer ")))
session = URI.decode_www_form(token["session"].as_s)
- scopes, expire, signature = validate_request(token, session, env.request, HMAC_KEY, nil)
+ scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil)
if email = Invidious::Database::SessionIDs.select_email(session)
user = Invidious::Database::Users.select!(email: email)
@@ -142,63 +143,8 @@ class APIHandler < Kemal::Handler
exclude ["/api/v1/auth/notifications"], "POST"
def call(env)
- return call_next env unless only_match? env
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- # Since /api/v1/notifications is an event-stream, we don't want
- # to wrap the response
- return call_next env if exclude_match? env
-
- # Here we swap out the socket IO so we can modify the response as needed
- output = env.response.output
- env.response.output = IO::Memory.new
-
- begin
- call_next env
-
- env.response.output.rewind
-
- if env.response.output.as(IO::Memory).size != 0 &&
- env.response.headers.includes_word?("Content-Type", "application/json")
- response = JSON.parse(env.response.output)
-
- if fields_text = env.params.query["fields"]?
- begin
- JSONFilter.filter(response, fields_text)
- rescue ex
- env.response.status_code = 400
- response = {"error" => ex.message}
- end
- end
-
- if env.params.query["pretty"]?.try &.== "1"
- response = response.to_pretty_json
- else
- response = response.to_json
- end
- else
- response = env.response.output.gets_to_end
- end
- rescue ex
- env.response.content_type = "application/json" if env.response.headers.includes_word?("Content-Type", "text/html")
- env.response.status_code = 500
-
- if env.response.headers.includes_word?("Content-Type", "application/json")
- response = {"error" => ex.message || "Unspecified error"}
-
- if env.params.query["pretty"]?.try &.== "1"
- response = response.to_pretty_json
- else
- response = response.to_json
- end
- end
- ensure
- env.response.output = output
- env.response.print response
-
- env.response.flush
- end
+ env.response.headers["Access-Control-Allow-Origin"] = "*" if only_match?(env)
+ call_next env
end
end
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr
index c3b53339..6add0237 100644
--- a/src/invidious/helpers/helpers.cr
+++ b/src/invidious/helpers/helpers.cr
@@ -22,31 +22,6 @@ struct Annotation
property annotations : String
end
-def login_req(f_req)
- data = {
- # Unfortunately there's not much information available on `bgRequest`; part of Google's BotGuard
- # Generally this is much longer (>1250 characters), see also
- # https://github.com/ytdl-org/youtube-dl/commit/baf67a604d912722b0fe03a40e9dc5349a2208cb .
- # For now this can be empty.
- "bgRequest" => %|["identifier",""]|,
- "pstMsg" => "1",
- "checkConnection" => "youtube",
- "checkedDomains" => "youtube",
- "hl" => "en",
- "deviceinfo" => %|[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]|,
- "f.req" => f_req,
- "flowName" => "GlifWebSignIn",
- "flowEntry" => "ServiceLogin",
- # "cookiesDisabled" => "false",
- # "gmscoreversion" => "undefined",
- # "continue" => "https://accounts.google.com/ManageAccount",
- # "azt" => "",
- # "bgHash" => "",
- }
-
- return HTTP::Params.encode(data)
-end
-
def html_to_content(description_html : String)
description = description_html.gsub(/(
)|(
": "\n", @@ -103,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -138,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -180,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -233,3 +181,20 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end + +# Fetch the playback requests tracker from the statistics endpoint. +# +# Creates a new tracker when unavailable. +def get_playback_statistic + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? + tracker = { + "totalRequests" => 0_i64, + "successfulRequests" => 0_i64, + "ratio" => 0_f64, + } + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker + end + + return tracker.as(Hash(String, Int64 | Float64)) +end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 39e183f2..bca2edda 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,19 +1,35 @@ -# "bn_BD" => load_locale("bn_BD"), # Bengali (Bangladesh) [Incomplete] -# "eu" => load_locale("eu"), # Basque [Incomplete] -# "sk" => load_locale("sk"), # Slovak [Incomplete] +# Languages requiring a better level of translation (at least 20%) +# to be added to the list below: +# +# "af" => "", # Afrikaans +# "az" => "", # Azerbaijani +# "be" => "", # Belarusian +# "bn_BD" => "", # Bengali (Bangladesh) +# "ia" => "", # Interlingua +# "or" => "", # Odia +# "tk" => "", # Turkmen +# "tok => "", # Toki Pona +# LOCALES_LIST = { "ar" => "العربية", # Arabic + "bg" => "български", # Bulgarian + "bn" => "বাংলা", # Bengali + "ca" => "Català", # Catalan "cs" => "Čeština", # Czech + "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek "en-US" => "English", # English "eo" => "Esperanto", # Esperanto "es" => "Español", # Spanish + "et" => "Eesti keel", # Estonian + "eu" => "Euskara", # Basque "fa" => "فارسی", # Persian "fi" => "Suomi", # Finnish "fr" => "Français", # French "he" => "עברית", # Hebrew + "hi" => "हिन्दी", # Hindi "hr" => "Hrvatski", # Croatian "hu-HU" => "Magyar Nyelv", # Hungarian "id" => "Bahasa Indonesia", # Indonesian @@ -21,6 +37,7 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean + "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch @@ -29,11 +46,15 @@ LOCALES_LIST = { "pt-BR" => "Português Brasileiro", # Portuguese (Brazil) "pt-PT" => "Português de Portugal", # Portuguese (Portugal) "ro" => "Română", # Romanian - "ru" => "русский", # Russian + "ru" => "Русский", # Russian + "si" => "සිංහල", # Sinhala + "sk" => "Slovenčina", # Slovak + "sl" => "Slovenščina", # Slovenian "sq" => "Shqip", # Albanian - "sr" => "srpski (latinica)", # Serbian (Latin) - "sr_Cyrl" => "српски (ћирилица)", # Serbian (Cyrillic) + "sr" => "Srpski (latinica)", # Serbian (Latin) + "sr_Cyrl" => "Српски (ћирилица)", # Serbian (Cyrillic) "sv-SE" => "Svenska", # Swedish + "ta" => "தமிழ்", # Tamil "tr" => "Türkçe", # Turkish "uk" => "Українська", # Ukrainian "vi" => "Tiếng Việt", # Vietnamese @@ -73,7 +94,7 @@ def load_all_locales return locales end -def translate(locale : String?, key : String, text : String | Nil = nil) : String +def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String # Log a warning if "key" doesn't exist in en-US locale and return # that key as the text, so this is more or less transparent to the user. if !LOCALES["en-US"].has_key?(key) @@ -96,10 +117,12 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin match_length = 0 raw_data.as_h.each do |hash_key, value| - if md = text.try &.match(/#{hash_key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size + if text.is_a?(String) + if md = text.try &.match(/#{hash_key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size + end end end end @@ -109,8 +132,13 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin raise "Invalid translation \"#{raw_data}\"" end - if text + if text.is_a?(String) translation = translation.gsub("`x`", text) + elsif text.is_a?(Hash(String, String)) + # adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic + text.each_key do |hash_key| + translation = translation.gsub("{{#{hash_key}}}", text[hash_key]) + end end return translation @@ -160,3 +188,12 @@ def translate_bool(locale : String?, translation : Bool) return translate(locale, "No") end end + +def locale_is_rtl?(locale : String?) + # Fallback to en-US + return false if locale.nil? + + # Arabic, Persian, Hebrew + # See https://en.wikipedia.org/wiki/Right-to-left_script#List_of_RTL_scripts + return {"ar", "fa", "he"}.includes? locale +end diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index e84f88fb..684e6d14 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -35,27 +35,35 @@ module I18next::Plurals Special_Slovenian = 21 Special_Hebrew = 22 Special_Odia = 23 + + # Mixed v3/v4 rules in Weblate + # `es`, `pt` and `pt-PT` doesn't seem to have been refreshed + # by weblate yet, but I suspect it will happen one day. + # See: https://github.com/translate/translate/issues/4873 + Special_French_Portuguese + Special_Hungarian_Serbian + Special_Spanish_Italian end private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fil", "fr", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt", "pt-BR", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", - "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", - "hu", "hy", "ia", "it", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", + "eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", - "ps", "pt-PT", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", + "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ - "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ - "be", "bs", "cnr", "dz", "hr", "ru", "sr", "uk", + "be", "bs", "cnr", "dz", "ru", "uk", ], } @@ -81,6 +89,13 @@ module I18next::Plurals "ro" => PluralForms::Special_Romanian, "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, + # Mixed v3/v4 rules + "es" => PluralForms::Special_Spanish_Italian, + "fr" => PluralForms::Special_French_Portuguese, + "hr" => PluralForms::Special_Hungarian_Serbian, + "it" => PluralForms::Special_Spanish_Italian, + "pt" => PluralForms::Special_French_Portuguese, + "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -150,9 +165,8 @@ module I18next::Plurals end def get_plural_form(locale : String) : PluralForms - # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code, - # except for pt-BR and pt-PT which needs to be kept as-is. - if !locale.matches?(/^pt-(BR|PT)$/) + # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code + if !locale.matches?(/^pt-PT$/) locale = locale.split('-')[0] end @@ -174,7 +188,7 @@ module I18next::Plurals # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check # from original i18next code - private def is_simple_plural(form : PluralForms) : Bool + private def simple_plural?(form : PluralForms) : Bool case form when .single_gt_one? then return true when .single_not_one? then return true @@ -196,7 +210,7 @@ module I18next::Plurals idx = SuffixIndex.get_index(plural_form, count) # Simple plurals are handled differently in all versions (but v4) - if @simplify_plural_suffix && is_simple_plural(plural_form) + if @simplify_plural_suffix && simple_plural?(plural_form) return (idx == 1) ? "_plural" : "" end @@ -246,6 +260,10 @@ module I18next::Plurals when .special_slovenian? then return special_slovenian(count) when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) + # Mixed v3/v4 forms + when .special_spanish_italian? then return special_cldr_spanish_italian(count) + when .special_french_portuguese? then return special_cldr_french_portuguese(count) + when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count) else # default, if nothing matched above return 0_u8 @@ -507,5 +525,42 @@ module I18next::Plurals def self.special_odia(count : Int) : UInt8 return (count == 1) ? 0_u8 : 1_u8 end + + # ------------------- + # "v3.5" rules + # ------------------- + + # Plural form for Spanish & Italian languages + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_spanish_italian(count : Int) : UInt8 + return 0_u8 if (count == 1) # one + return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many + return 2_u8 # other + end + + # Plural form for French and Portuguese + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_french_portuguese(count : Int) : UInt8 + return 0_u8 if (count == 0 || count == 1) # one + return 1_u8 if (count % 1_000_000 == 0) # many + return 2_u8 # other + end + + # Plural form for Hungarian and Serbian + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_hungarian_serbian(count : Int) : UInt8 + n_mod_10 = count % 10 + n_mod_100 = count % 100 + + return 0_u8 if (n_mod_10 == 1 && n_mod_100 != 11) # one + return 1_u8 if (2 <= n_mod_10 <= 4 && (n_mod_100 < 12 || 14 < n_mod_100)) # few + return 2_u8 # other + end end end diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr deleted file mode 100644 index b8e8f96d..00000000 --- a/src/invidious/helpers/json_filter.cr +++ /dev/null @@ -1,248 +0,0 @@ -module JSONFilter - alias BracketIndex = Hash(Int64, Int64) - - alias GroupedFieldsValue = String | Array(GroupedFieldsValue) - alias GroupedFieldsList = Array(GroupedFieldsValue) - - class FieldsParser - class ParseError < Exception - end - - # Returns the `Regex` pattern used to match nest groups - def self.nest_group_pattern : Regex - # uses a '.' character to match json keys as they are allowed - # to contain any unicode codepoint - /(?:|,)(?[^,\n]*?)\(/
- end
-
- # Returns the `Regex` pattern used to check if there are any empty nest groups
- def self.unnamed_nest_group_pattern : Regex
- /^\(|\(\(|\/\(/
- end
-
- def self.parse_fields(fields_text : String) : Nil
- if fields_text.empty?
- raise FieldsParser::ParseError.new "Fields is empty"
- end
-
- opening_bracket_count = fields_text.count('(')
- closing_bracket_count = fields_text.count(')')
-
- if opening_bracket_count != closing_bracket_count
- bracket_type = opening_bracket_count > closing_bracket_count ? "opening" : "closing"
- raise FieldsParser::ParseError.new "There are too many #{bracket_type} brackets (#{opening_bracket_count}:#{closing_bracket_count})"
- elsif match_result = unnamed_nest_group_pattern.match(fields_text)
- raise FieldsParser::ParseError.new "Unnamed nest group at position #{match_result.begin}"
- end
-
- # first, handle top-level single nested properties: items/id, playlistItems/snippet, etc
- parse_single_nests(fields_text) { |nest_list| yield nest_list }
-
- # next, handle nest groups: items(id, etag, etc)
- parse_nest_groups(fields_text) { |nest_list| yield nest_list }
- end
-
- def self.parse_single_nests(fields_text : String) : Nil
- single_nests = remove_nest_groups(fields_text)
-
- if !single_nests.empty?
- property_nests = single_nests.split(',')
-
- property_nests.each do |nest|
- nest_list = nest.split('/')
- if nest_list.includes? ""
- raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list}"
- end
- yield nest_list
- end
- # else
- # raise FieldsParser::ParseError.new "Empty key in nest list 22: #{fields_text} | #{single_nests}"
- end
- end
-
- def self.parse_nest_groups(fields_text : String) : Nil
- nest_stack = [] of NamedTuple(group_name: String, closing_bracket_index: Int64)
- bracket_pairs = get_bracket_pairs(fields_text, true)
-
- text_index = 0
- regex_index = 0
-
- while regex_result = self.nest_group_pattern.match(fields_text, regex_index)
- raw_match = regex_result[0]
- group_name = regex_result["groupname"]
-
- text_index = regex_result.begin
- regex_index = regex_result.end
-
- if text_index.nil? || regex_index.nil?
- raise FieldsParser::ParseError.new "Received invalid index while parsing nest groups: text_index: #{text_index} | regex_index: #{regex_index}"
- end
-
- offset = raw_match.starts_with?(',') ? 1 : 0
-
- opening_bracket_index = (text_index + group_name.size) + offset
- closing_bracket_index = bracket_pairs[opening_bracket_index]
- content_start = opening_bracket_index + 1
-
- content = fields_text[content_start...closing_bracket_index]
-
- if content.empty?
- raise FieldsParser::ParseError.new "Empty nest group at position #{content_start}"
- else
- content = remove_nest_groups(content)
- end
-
- while nest_stack.size > 0 && closing_bracket_index > nest_stack[nest_stack.size - 1][:closing_bracket_index]
- if nest_stack.size
- nest_stack.pop
- end
- end
-
- group_name.split('/').each do |name|
- nest_stack.push({
- group_name: name,
- closing_bracket_index: closing_bracket_index,
- })
- end
-
- if !content.empty?
- properties = content.split(',')
-
- properties.each do |prop|
- nest_list = nest_stack.map { |nest_prop| nest_prop[:group_name] }
-
- if !prop.empty?
- if prop.includes?('/')
- parse_single_nests(prop) { |list| nest_list += list }
- else
- nest_list.push prop
- end
- else
- raise FieldsParser::ParseError.new "Empty key in nest list: #{nest_list << prop}"
- end
-
- yield nest_list
- end
- end
- end
- end
-
- def self.remove_nest_groups(text : String) : String
- content_bracket_pairs = get_bracket_pairs(text, false)
-
- content_bracket_pairs.each_key.to_a.reverse.each do |opening_bracket|
- closing_bracket = content_bracket_pairs[opening_bracket]
- last_comma = text.rindex(',', opening_bracket) || 0
-
- text = text[0...last_comma] + text[closing_bracket + 1...text.size]
- end
-
- return text.starts_with?(',') ? text[1...text.size] : text
- end
-
- def self.get_bracket_pairs(text : String, recursive = true) : BracketIndex
- istart = [] of Int64
- bracket_index = BracketIndex.new
-
- text.each_char_with_index do |char, index|
- if char == '('
- istart.push(index.to_i64)
- end
-
- if char == ')'
- begin
- opening = istart.pop
- if recursive || (!recursive && istart.size == 0)
- bracket_index[opening] = index.to_i64
- end
- rescue
- raise FieldsParser::ParseError.new "No matching opening parenthesis at: #{index}"
- end
- end
- end
-
- if istart.size != 0
- idx = istart.pop
- raise FieldsParser::ParseError.new "No matching closing parenthesis at: #{idx}"
- end
-
- return bracket_index
- end
- end
-
- class FieldsGrouper
- alias SkeletonValue = Hash(String, SkeletonValue)
-
- def self.create_json_skeleton(fields_text : String) : SkeletonValue
- root_hash = {} of String => SkeletonValue
-
- FieldsParser.parse_fields(fields_text) do |nest_list|
- current_item = root_hash
- nest_list.each do |key|
- if current_item[key]?
- current_item = current_item[key]
- else
- current_item[key] = {} of String => SkeletonValue
- current_item = current_item[key]
- end
- end
- end
- root_hash
- end
-
- def self.create_grouped_fields_list(json_skeleton : SkeletonValue) : GroupedFieldsList
- grouped_fields_list = GroupedFieldsList.new
- json_skeleton.each do |key, value|
- grouped_fields_list.push key
-
- nested_keys = create_grouped_fields_list(value)
- grouped_fields_list.push nested_keys unless nested_keys.empty?
- end
- return grouped_fields_list
- end
- end
-
- class FilterError < Exception
- end
-
- def self.filter(item : JSON::Any, fields_text : String, in_place : Bool = true)
- skeleton = FieldsGrouper.create_json_skeleton(fields_text)
- grouped_fields_list = FieldsGrouper.create_grouped_fields_list(skeleton)
- filter(item, grouped_fields_list, in_place)
- end
-
- def self.filter(item : JSON::Any, grouped_fields_list : GroupedFieldsList, in_place : Bool = true) : JSON::Any
- item = item.clone unless in_place
-
- if !item.as_h? && !item.as_a?
- raise FilterError.new "Can't filter '#{item}' by #{grouped_fields_list}"
- end
-
- top_level_keys = Array(String).new
- grouped_fields_list.each do |value|
- if value.is_a? String
- top_level_keys.push value
- elsif value.is_a? Array
- if !top_level_keys.empty?
- key_to_filter = top_level_keys.last
-
- if item.as_h?
- filter(item[key_to_filter], value, in_place: true)
- elsif item.as_a?
- item.as_a.each { |arr_item| filter(arr_item[key_to_filter], value, in_place: true) }
- end
- else
- raise FilterError.new "Tried to filter while top level keys list is empty"
- end
- end
- end
-
- if item.as_h?
- item.as_h.select! top_level_keys
- elsif item.as_a?
- item.as_a.map { |value| filter(value, top_level_keys, in_place: true) }
- end
-
- item
- end
-end
diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr
index e2e50905..03349595 100644
--- a/src/invidious/helpers/logger.cr
+++ b/src/invidious/helpers/logger.cr
@@ -1,3 +1,5 @@
+require "colorize"
+
enum LogLevel
All = 0
Trace = 1
@@ -10,7 +12,9 @@ enum LogLevel
end
class Invidious::LogHandler < Kemal::BaseLogHandler
- def initialize(@io : IO = STDOUT, @level = LogLevel::Debug)
+ def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true)
+ Colorize.enabled = use_color
+ Colorize.on_tty_only!
end
def call(context : HTTP::Server::Context)
@@ -34,28 +38,27 @@ class Invidious::LogHandler < Kemal::BaseLogHandler
context
end
- def puts(message : String)
- @io << message << '\n'
- @io.flush
- end
-
def write(message : String)
@io << message
@io.flush
end
- def set_log_level(level : String)
- @level = LogLevel.parse(level)
- end
-
- def set_log_level(level : LogLevel)
- @level = level
+ def color(level)
+ case level
+ when LogLevel::Trace then :cyan
+ when LogLevel::Debug then :green
+ when LogLevel::Info then :white
+ when LogLevel::Warn then :yellow
+ when LogLevel::Error then :red
+ when LogLevel::Fatal then :magenta
+ else :default
+ end
end
{% for level in %w(trace debug info warn error fatal) %}
def {{level.id}}(message : String)
if LogLevel::{{level.id.capitalize}} >= @level
- puts("#{Time.utc} [{{level.id}}] #{message}")
+ puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})))
end
end
{% end %}
diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr
index 75df1612..84847321 100644
--- a/src/invidious/helpers/macros.cr
+++ b/src/invidious/helpers/macros.cr
@@ -48,13 +48,18 @@ module JSON::Serializable
end
end
-macro templated(filename, template = "template", navbar_search = true)
+macro templated(_filename, template = "template", navbar_search = true)
navbar_search = {{navbar_search}}
- render "src/invidious/views/#{{{filename}}}.ecr", "src/invidious/views/#{{{template}}}.ecr"
+
+ {{ filename = "src/invidious/views/" + _filename + ".ecr" }}
+ {{ layout = "src/invidious/views/" + template + ".ecr" }}
+
+ __content_filename__ = {{filename}}
+ render {{filename}}, {{layout}}
end
macro rendered(filename)
- render "src/invidious/views/#{{{filename}}}.ecr"
+ render("src/invidious/views/#{{{filename}}}.ecr")
end
# Similar to Kemals halt method but works in a
diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr
index bfbc237c..2796a8dc 100644
--- a/src/invidious/helpers/serialized_yt_data.cr
+++ b/src/invidious/helpers/serialized_yt_data.cr
@@ -1,3 +1,16 @@
+@[Flags]
+enum VideoBadges
+ LiveNow
+ Premium
+ ThreeD
+ FourK
+ New
+ EightK
+ VR180
+ VR360
+ ClosedCaptions
+end
+
struct SearchVideo
include DB::Serializable
@@ -9,9 +22,10 @@ struct SearchVideo
property views : Int64
property description_html : String
property length_seconds : Int32
- property live_now : Bool
- property premium : Bool
property premiere_timestamp : Time?
+ property author_verified : Bool
+ property author_thumbnail : String?
+ property badges : VideoBadges
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
@@ -73,25 +87,52 @@ struct SearchVideo
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
+ json.field "authorVerified", self.author_verified
+
+ author_thumbnail = self.author_thumbnail
+
+ if author_thumbnail
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+ end
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id)
+ Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views
+ json.field "viewCountText", translate_count(locale, "generic_views_count", self.views, NumberFormatting::Short)
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds
- json.field "liveNow", self.live_now
- json.field "premium", self.premium
- json.field "isUpcoming", self.is_upcoming
+ json.field "liveNow", self.badges.live_now?
+ json.field "premium", self.badges.premium?
+ json.field "isUpcoming", self.upcoming?
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
+ json.field "isNew", self.badges.new?
+ json.field "is4k", self.badges.four_k?
+ json.field "is8k", self.badges.eight_k?
+ json.field "isVr180", self.badges.vr180?
+ json.field "isVr360", self.badges.vr360?
+ json.field "is3d", self.badges.three_d?
+ json.field "hasCaptions", self.badges.closed_captions?
end
end
@@ -106,7 +147,7 @@ struct SearchVideo
to_json(nil, json)
end
- def is_upcoming
+ def upcoming?
premiere_timestamp ? true : false
end
end
@@ -129,6 +170,7 @@ struct SearchPlaylist
property video_count : Int32
property videos : Array(SearchPlaylistVideo)
property thumbnail : String?
+ property author_verified : Bool
def to_json(locale : String?, json : JSON::Builder)
json.object do
@@ -141,6 +183,8 @@ struct SearchPlaylist
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
+ json.field "authorVerified", self.author_verified
+
json.field "videoCount", self.video_count
json.field "videos" do
json.array do
@@ -151,7 +195,7 @@ struct SearchPlaylist
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
- generate_thumbnails(json, video.id)
+ Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
end
end
@@ -180,8 +224,10 @@ struct SearchChannel
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
+ property channel_handle : String?
property description_html : String
property auto_generated : Bool
+ property author_verified : Bool
def to_json(locale : String?, json : JSON::Builder)
json.object do
@@ -189,14 +235,14 @@ struct SearchChannel
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
-
+ json.field "authorVerified", self.author_verified
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
- json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
+ json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
@@ -207,6 +253,7 @@ struct SearchChannel
json.field "autoGenerated", self.auto_generated
json.field "subCount", self.subscriber_count
json.field "videoCount", self.video_count
+ json.field "channelHandle", self.channel_handle
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
@@ -225,6 +272,74 @@ struct SearchChannel
end
end
+struct SearchHashtag
+ include DB::Serializable
+
+ property title : String
+ property url : String
+ property video_count : Int64
+ property channel_count : Int64
+
+ def to_json(locale : String?, json : JSON::Builder)
+ json.object do
+ json.field "type", "hashtag"
+ json.field "title", self.title
+ json.field "url", self.url
+ json.field "videoCount", self.video_count
+ json.field "channelCount", self.channel_count
+ end
+ end
+end
+
+# A `ProblematicTimelineItem` is a `SearchItem` created by Invidious that
+# represents an item that caused an exception during parsing.
+#
+# This is not a parsed object from YouTube but rather an Invidious-only type
+# created to gracefully communicate parse errors without throwing away
+# the rest of the (hopefully) successfully parsed item on a page.
+struct ProblematicTimelineItem
+ property parse_exception : Exception
+ property id : String
+
+ def initialize(@parse_exception)
+ @id = Random.new.hex(8)
+ end
+
+ def to_json(locale : String?, json : JSON::Builder)
+ json.object do
+ json.field "type", "parse-error"
+ json.field "errorMessage", @parse_exception.message
+ json.field "errorBacktrace", @parse_exception.inspect_with_backtrace
+ end
+ end
+
+ # Provides compatibility with PlaylistVideo
+ def to_json(json : JSON::Builder, *args, **kwargs)
+ return to_json("", json)
+ end
+
+ def to_xml(env, locale, xml : XML::Builder)
+ xml.element("entry") do
+ xml.element("id") { xml.text "iv-err-#{@id}" }
+ xml.element("title") { xml.text "Parse Error: This item has failed to parse" }
+ xml.element("updated") { xml.text Time.utc.to_rfc3339 }
+
+ xml.element("content", type: "xhtml") do
+ xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
+ xml.element("div") do
+ xml.element("h4") { translate(locale, "timeline_parse_error_placeholder_heading") }
+ xml.element("p") { translate(locale, "timeline_parse_error_placeholder_message") }
+ end
+
+ xml.element("pre") do
+ get_issue_template(env, @parse_exception)
+ end
+ end
+ end
+ end
+ end
+end
+
class Category
include DB::Serializable
@@ -260,4 +375,11 @@ class Category
end
end
-alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category
+struct Continuation
+ getter token
+
+ def initialize(@token : String)
+ end
+end
+
+alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category | ProblematicTimelineItem
diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr
new file mode 100644
index 00000000..6d198a42
--- /dev/null
+++ b/src/invidious/helpers/sig_helper.cr
@@ -0,0 +1,349 @@
+require "uri"
+require "socket"
+require "socket/tcp_socket"
+require "socket/unix_socket"
+
+{% if flag?(:advanced_debug) %}
+ require "io/hexdump"
+{% end %}
+
+private alias NetworkEndian = IO::ByteFormat::NetworkEndian
+
+module Invidious::SigHelper
+ enum UpdateStatus
+ Updated
+ UpdateNotRequired
+ Error
+ end
+
+ # -------------------
+ # Payload types
+ # -------------------
+
+ abstract struct Payload
+ end
+
+ struct StringPayload < Payload
+ getter string : String
+
+ def initialize(str : String)
+ raise Exception.new("SigHelper: String can't be empty") if str.empty?
+ @string = str
+ end
+
+ def self.from_bytes(slice : Bytes)
+ size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice)
+ if size == 0 # Error code
+ raise Exception.new("SigHelper: Server encountered an error")
+ end
+
+ if (slice.bytesize - 2) != size
+ raise Exception.new("SigHelper: String size mismatch")
+ end
+
+ if str = String.new(slice[2..])
+ return self.new(str)
+ else
+ raise Exception.new("SigHelper: Can't read string from socket")
+ end
+ end
+
+ def to_io(io)
+ # `.to_u16` raises if there is an overflow during the conversion
+ io.write_bytes(@string.bytesize.to_u16, NetworkEndian)
+ io.write(@string.to_slice)
+ end
+ end
+
+ private enum Opcode
+ FORCE_UPDATE = 0
+ DECRYPT_N_SIGNATURE = 1
+ DECRYPT_SIGNATURE = 2
+ GET_SIGNATURE_TIMESTAMP = 3
+ GET_PLAYER_STATUS = 4
+ PLAYER_UPDATE_TIMESTAMP = 5
+ end
+
+ private record Request,
+ opcode : Opcode,
+ payload : Payload?
+
+ # ----------------------
+ # High-level functions
+ # ----------------------
+
+ class Client
+ @mux : Multiplexor
+
+ def initialize(uri_or_path)
+ @mux = Multiplexor.new(uri_or_path)
+ end
+
+ # Forces the server to re-fetch the YouTube player, and extract the necessary
+ # components from it (nsig function code, sig function code, signature timestamp).
+ def force_update : UpdateStatus
+ request = Request.new(Opcode::FORCE_UPDATE, nil)
+
+ value = send_request(request) do |bytes|
+ IO::ByteFormat::NetworkEndian.decode(UInt16, bytes)
+ end
+
+ case value
+ when 0x0000 then return UpdateStatus::Error
+ when 0xFFFF then return UpdateStatus::UpdateNotRequired
+ when 0xF44F then return UpdateStatus::Updated
+ else
+ code = value.nil? ? "nil" : value.to_s(base: 16)
+ raise Exception.new("SigHelper: Invalid status code received #{code}")
+ end
+ end
+
+ # Decrypt a provided n signature using the server's current nsig function
+ # code, and return the result (or an error).
+ def decrypt_n_param(n : String) : String?
+ request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n))
+
+ n_dec = self.send_request(request) do |bytes|
+ StringPayload.from_bytes(bytes).string
+ end
+
+ return n_dec
+ end
+
+ # Decrypt a provided s signature using the server's current sig function
+ # code, and return the result (or an error).
+ def decrypt_sig(sig : String) : String?
+ request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig))
+
+ sig_dec = self.send_request(request) do |bytes|
+ StringPayload.from_bytes(bytes).string
+ end
+
+ return sig_dec
+ end
+
+ # Return the signature timestamp from the server's current player
+ def get_signature_timestamp : UInt64?
+ request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil)
+
+ return self.send_request(request) do |bytes|
+ IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)
+ end
+ end
+
+ # Return the current player's version
+ def get_player : UInt32?
+ request = Request.new(Opcode::GET_PLAYER_STATUS, nil)
+
+ return self.send_request(request) do |bytes|
+ has_player = (bytes[0] == 0xFF)
+ player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4])
+ has_player ? player_version : nil
+ end
+ end
+
+ # Return when the player was last updated
+ def get_player_timestamp : UInt64?
+ request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil)
+
+ return self.send_request(request) do |bytes|
+ IO::ByteFormat::NetworkEndian.decode(UInt64, bytes)
+ end
+ end
+
+ private def send_request(request : Request, &)
+ channel = @mux.send(request)
+ slice = channel.receive
+ return yield slice
+ rescue ex
+ LOGGER.debug("SigHelper: Error when sending a request")
+ LOGGER.trace(ex.inspect_with_backtrace)
+ return nil
+ end
+ end
+
+ # ---------------------
+ # Low level functions
+ # ---------------------
+
+ class Multiplexor
+ alias TransactionID = UInt32
+ record Transaction, channel = ::Channel(Bytes).new
+
+ @prng = Random.new
+ @mutex = Mutex.new
+ @queue = {} of TransactionID => Transaction
+
+ @conn : Connection
+ @uri_or_path : String
+
+ def initialize(@uri_or_path)
+ @conn = Connection.new(uri_or_path)
+ listen
+ end
+
+ def listen : Nil
+ raise "Socket is closed" if @conn.closed?
+
+ LOGGER.debug("SigHelper: Multiplexor listening")
+
+ spawn do
+ loop do
+ begin
+ receive_data
+ rescue ex
+ LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...")
+ # We close the socket because for some reason is not closed.
+ @conn.close
+ loop do
+ begin
+ @conn = Connection.new(@uri_or_path)
+ LOGGER.info("SigHelper: Reconnected to SigHelper!")
+ rescue ex
+ LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")
+ sleep 500.milliseconds
+ next
+ end
+ break if !@conn.closed?
+ end
+ end
+ Fiber.yield
+ end
+ end
+ end
+
+ def send(request : Request)
+ transaction = Transaction.new
+ transaction_id = @prng.rand(TransactionID)
+
+ # Add transaction to queue
+ @mutex.synchronize do
+ # On a 32-bits random integer, this should never happen. Though, just in case, ...
+ if @queue[transaction_id]?
+ raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!")
+ end
+
+ @queue[transaction_id] = transaction
+ end
+
+ write_packet(transaction_id, request)
+
+ return transaction.channel
+ end
+
+ def receive_data
+ transaction_id, slice = read_packet
+
+ @mutex.synchronize do
+ if transaction = @queue.delete(transaction_id)
+ # Remove transaction from queue and send data to the channel
+ transaction.channel.send(slice)
+ LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel")
+ else
+ raise Exception.new("SigHelper: Received transaction was not in queue")
+ end
+ end
+ end
+
+ # Read a single packet from the socket
+ private def read_packet : {TransactionID, Bytes}
+ # Header
+ transaction_id = @conn.read_bytes(UInt32, NetworkEndian)
+ length = @conn.read_bytes(UInt32, NetworkEndian)
+
+ LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}")
+
+ if length > 67_000
+ raise Exception.new("SigHelper: Packet longer than expected (#{length})")
+ end
+
+ # Payload
+ slice = Bytes.new(length)
+ @conn.read(slice) if length > 0
+
+ LOGGER.trace("SigHelper: payload = #{slice}")
+ LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done")
+
+ return transaction_id, slice
+ end
+
+ # Write a single packet to the socket
+ private def write_packet(transaction_id : TransactionID, request : Request)
+ LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}")
+
+ io = IO::Memory.new(1024)
+ io.write_bytes(request.opcode.to_u8, NetworkEndian)
+ io.write_bytes(transaction_id, NetworkEndian)
+
+ if payload = request.payload
+ payload.to_io(io)
+ end
+
+ @conn.send(io)
+ @conn.flush
+
+ LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done")
+ end
+ end
+
+ class Connection
+ @socket : UNIXSocket | TCPSocket
+
+ {% if flag?(:advanced_debug) %}
+ @io : IO::Hexdump
+ {% end %}
+
+ def initialize(host_or_path : String)
+ case host_or_path
+ when .starts_with?('/')
+ # Make sure that the file exists
+ if File.exists?(host_or_path)
+ @socket = UNIXSocket.new(host_or_path)
+ else
+ raise Exception.new("SigHelper: '#{host_or_path}' no such file")
+ end
+ when .starts_with?("tcp://")
+ uri = URI.parse(host_or_path)
+ @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)
+ else
+ uri = URI.parse("tcp://#{host_or_path}")
+ @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!)
+ end
+ LOGGER.info("SigHelper: Using helper at '#{host_or_path}'")
+
+ {% if flag?(:advanced_debug) %}
+ @io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true)
+ {% end %}
+
+ @socket.sync = false
+ @socket.blocking = false
+ end
+
+ def closed? : Bool
+ return @socket.closed?
+ end
+
+ def close : Nil
+ @socket.close if !@socket.closed?
+ end
+
+ def flush(*args, **options)
+ @socket.flush(*args, **options)
+ end
+
+ def send(*args, **options)
+ @socket.send(*args, **options)
+ end
+
+ # Wrap IO functions, with added debug tooling if needed
+ {% for function in %w(read read_bytes write write_bytes) %}
+ def {{function.id}}(*args, **options)
+ {% if flag?(:advanced_debug) %}
+ @io.{{function.id}}(*args, **options)
+ {% else %}
+ @socket.{{function.id}}(*args, **options)
+ {% end %}
+ end
+ {% end %}
+ end
+end
diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr
index ee09415b..82a28fc0 100644
--- a/src/invidious/helpers/signatures.cr
+++ b/src/invidious/helpers/signatures.cr
@@ -1,73 +1,53 @@
-alias SigProc = Proc(Array(String), Int32, Array(String))
+require "http/params"
+require "./sig_helper"
-struct DecryptFunction
- @decrypt_function = [] of {SigProc, Int32}
- @decrypt_time = Time.monotonic
+class Invidious::DecryptFunction
+ @last_update : Time = Time.utc - 42.days
- def initialize(@use_polling = true)
+ def initialize(uri_or_path)
+ @client = SigHelper::Client.new(uri_or_path)
+ self.check_update
end
- def update_decrypt_function
- @decrypt_function = fetch_decrypt_function
+ def check_update
+ # If we have updated in the last 5 minutes, do nothing
+ return if (Time.utc - @last_update) < 5.minutes
+
+ # Get the amount of time elapsed since when the player was updated, in the
+ # event where multiple invidious processes are run in parallel.
+ update_time_elapsed = (@client.get_player_timestamp || 301).seconds
+
+ if update_time_elapsed > 5.minutes
+ LOGGER.debug("Signature: Player might be outdated, updating")
+ @client.force_update
+ @last_update = Time.utc
+ end
end
- private def fetch_decrypt_function(id = "CvFH_6DNRCY")
- document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
- url = document.match(/src="(?\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
- player = YT_POOL.client &.get(url).body
-
- function_name = player.match(/^(?[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
- function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?[^}]+)}/m).not_nil!["body"]
- function_body = function_body.split(";")[1..-2]
-
- var_name = function_body[0][0, 2]
- var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?(.*?))};/).not_nil!["body"]
-
- operations = {} of String => SigProc
- var_body.split("},").each do |operation|
- op_name = operation.match(/^[^:]+/).not_nil![0]
- op_body = operation.match(/\{[^}]+/).not_nil![0]
-
- case op_body
- when "{a.reverse()"
- operations[op_name] = ->(a : Array(String), _b : Int32) { a.reverse }
- when "{a.splice(0,b)"
- operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
- else
- operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
- end
- end
-
- decrypt_function = [] of {SigProc, Int32}
- function_body.each do |function|
- function = function.lchop(var_name).delete("[].")
-
- op_name = function.match(/[^\(]+/).not_nil![0]
- value = function.match(/\(\w,(?[\d]+)\)/).not_nil!["value"].to_i
-
- decrypt_function << {operations[op_name], value}
- end
-
- return decrypt_function
+ def decrypt_nsig(n : String) : String?
+ self.check_update
+ return @client.decrypt_n_param(n)
+ rescue ex
+ LOGGER.debug(ex.message || "Signature: Unknown error")
+ LOGGER.trace(ex.inspect_with_backtrace)
+ return nil
end
- def decrypt_signature(fmt : Hash(String, JSON::Any))
- return "" if !fmt["s"]? || !fmt["sp"]?
+ def decrypt_signature(str : String) : String?
+ self.check_update
+ return @client.decrypt_sig(str)
+ rescue ex
+ LOGGER.debug(ex.message || "Signature: Unknown error")
+ LOGGER.trace(ex.inspect_with_backtrace)
+ return nil
+ end
- sp = fmt["sp"].as_s
- sig = fmt["s"].as_s.split("")
- if !@use_polling
- now = Time.monotonic
- if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0
- @decrypt_function = fetch_decrypt_function
- @decrypt_time = Time.monotonic
- end
- end
-
- @decrypt_function.each do |proc, value|
- sig = proc.call(sig, value)
- end
-
- return "&#{sp}=#{sig.join("")}"
+ def get_sts : UInt64?
+ self.check_update
+ return @client.get_signature_timestamp
+ rescue ex
+ LOGGER.debug(ex.message || "Signature: Unknown error")
+ LOGGER.trace(ex.inspect_with_backtrace)
+ return nil
end
end
diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr
index c1dc17db..5637e533 100644
--- a/src/invidious/helpers/utils.cr
+++ b/src/invidious/helpers/utils.cr
@@ -52,9 +52,9 @@ def recode_length_seconds(time)
end
def decode_interval(string : String) : Time::Span
- rawMinutes = string.try &.to_i32?
+ raw_minutes = string.try &.to_i32?
- if !rawMinutes
+ if !raw_minutes
hours = /(?\d+)h/.match(string).try &.["hours"].try &.to_i32
hours ||= 0
@@ -63,7 +63,7 @@ def decode_interval(string : String) : Time::Span
time = Time::Span.new(hours: hours, minutes: minutes)
else
- time = Time::Span.new(minutes: rawMinutes)
+ time = Time::Span.new(minutes: raw_minutes)
end
return time
@@ -111,24 +111,27 @@ def decode_date(string : String)
else nil # Continue
end
- # String matches format "20 hours ago", "4 months ago"...
- date = string.split(" ")[-3, 3]
- delta = date[0].to_i
+ # String matches format "20 hours ago", "4 months ago", "20s ago", "15min ago"...
+ match = string.match(/(?\d+) ?(?[smhdwy]\w*) ago/)
- case date[1]
- when .includes? "second"
+ raise "Could not parse #{string}" if match.nil?
+
+ delta = match["count"].to_i
+
+ case match["span"]
+ when .starts_with? "s" # second(s)
delta = delta.seconds
- when .includes? "minute"
+ when .starts_with? "mi" # minute(s)
delta = delta.minutes
- when .includes? "hour"
+ when .starts_with? "h" # hour(s)
delta = delta.hours
- when .includes? "day"
+ when .starts_with? "d" # day(s)
delta = delta.days
- when .includes? "week"
+ when .starts_with? "w" # week(s)
delta = delta.weeks
- when .includes? "month"
+ when .starts_with? "mo" # month(s)
delta = delta.months
- when .includes? "year"
+ when .starts_with? "y" # year(s)
delta = delta.years
else
raise "Could not parse #{string}"
@@ -161,21 +164,19 @@ def number_with_separator(number)
number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse
end
-def short_text_to_number(short_text : String) : Int32
- case short_text
- when .ends_with? "M"
- number = short_text.rstrip(" mM").to_f
- number *= 1000000
- when .ends_with? "K"
- number = short_text.rstrip(" kK").to_f
- number *= 1000
- else
- number = short_text.rstrip(" ")
+def short_text_to_number(short_text : String) : Int64
+ matches = /(?\d+(\.\d+)?)\s?(?[mMkKbB]?)/.match(short_text)
+ number = matches.try &.["number"].to_f || 0.0
+
+ case matches.try &.["suffix"].downcase
+ when "k" then number *= 1_000
+ when "m" then number *= 1_000_000
+ when "b" then number *= 1_000_000_000
end
- number = number.to_i
-
- return number
+ return number.to_i64
+rescue ex
+ return 0_i64
end
def number_to_short_text(number)
@@ -261,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true)
end
referer = referer.request_target
- referer = "/" + referer.gsub(/[^\/?@&%=\-_.0-9a-zA-Z]/, "").lstrip("/\\")
+ referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z+]/, "").lstrip("/\\")
if referer == env.request.path
referer = fallback
@@ -322,64 +323,82 @@ def parse_range(range)
return 0_i64, nil
end
-def fetch_random_instance
- begin
- instance_api_client = make_client(URI.parse("https://api.invidious.io"))
-
- # Timeouts
- instance_api_client.connect_timeout = 10.seconds
- instance_api_client.dns_timeout = 10.seconds
-
- instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a
- instance_api_client.close
- rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException
- instance_list = [] of JSON::Any
+def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "…") : String
+ str = uri.to_s.sub(/^https?:\/\//, "")
+ if str.size > max_length
+ str = "#{str[0, max_length]}#{suffix}"
end
+ return str
+end
- filtered_instance_list = [] of String
+# Get the html link from a NavigationEndpoint or an innertubeCommand
+def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
+ if url = endpoint.dig?("urlEndpoint", "url").try &.as_s
+ url = URI.parse(url)
+ displayed_url = text
- instance_list.each do |data|
- # TODO Check if current URL is onion instance and use .onion types if so.
- if data[1]["type"] == "https"
- # Instances can have statistics disabled, which is an requirement of version validation.
- # as_nil? doesn't exist. Thus we'll have to handle the error raised if as_nil fails.
- begin
- data[1]["stats"].as_nil
- next
- rescue TypeCastError
- end
-
- # stats endpoint could also lack the software dict.
- next if data[1]["stats"]["software"]?.nil?
-
- # Makes sure the instance isn't too outdated.
- if remote_version = data[1]["stats"]?.try &.["software"]?.try &.["version"]
- remote_commit_date = remote_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/)
- next if !remote_commit_date
-
- remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC)
- local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC)
-
- next if (remote_commit_date - local_commit_date).abs.days > 30
-
- begin
- data[1]["monitor"].as_nil
- health = data[1]["monitor"].as_h["dailyRatios"][0].as_h["ratio"]
- filtered_instance_list << data[0].as_s if health.to_s.to_f > 90
- rescue TypeCastError
- # We can't check the health if the monitoring is broken. Thus we'll just add it to the list
- # and move on. Ideally we'll ignore any instance that has broken health monitoring but due to the fact that
- # it's an error that often occurs with all the instances at the same time, we have to just skip the check.
- filtered_instance_list << data[0].as_s
- end
+ if url.host == "youtu.be"
+ url = "/watch?v=#{url.request_target.lstrip('/')}"
+ elsif url.host.nil? || url.host.not_nil!.ends_with?("youtube.com")
+ if url.path == "/redirect"
+ # Sometimes, links can be corrupted (why?) so make sure to fallback
+ # nicely. See https://github.com/iv-org/invidious/issues/2682
+ url = url.query_params["q"]? || ""
+ displayed_url = url
+ else
+ url = url.request_target
+ displayed_url = "youtube.com#{url}"
end
end
- end
- # If for some reason no instances managed to get fetched successfully then we'll just redirect to redirect.invidious.io
- if filtered_instance_list.size == 0
- return "redirect.invidious.io"
- end
+ text = %(#{reduce_uri(displayed_url)})
+ elsif watch_endpoint = endpoint.dig?("watchEndpoint")
+ start_time = watch_endpoint["startTimeSeconds"]?.try &.as_i
+ link_video_id = watch_endpoint["videoId"].as_s
- return filtered_instance_list.sample(1)[0]
+ url = "/watch?v=#{link_video_id}"
+ url += "&t=#{start_time}" if !start_time.nil?
+
+ # If the current video ID (passed through from the caller function)
+ # is the same as the video ID in the link, add HTML attributes for
+ # the JS handler function that bypasses page reload.
+ #
+ # See: https://github.com/iv-org/invidious/issues/3063
+ if link_video_id == video_id
+ start_time ||= 0
+ text = %(#{reduce_uri(text)})
+ else
+ text = %(#{text})
+ end
+ elsif url = endpoint.dig?("commandMetadata", "webCommandMetadata", "url").try &.as_s
+ if text.starts_with?(/\s?[@#]/)
+ # Handle "pings" in comments and hasthags differently
+ # See:
+ # - https://github.com/iv-org/invidious/issues/3038
+ # - https://github.com/iv-org/invidious/issues/3062
+ text = %(#{text})
+ else
+ text = %(#{reduce_uri(text)})
+ end
+ end
+ return text
+end
+
+def encrypt_ecb_without_salt(data, key)
+ cipher = OpenSSL::Cipher.new("aes-128-ecb")
+ cipher.encrypt
+ cipher.key = key
+
+ io = IO::Memory.new
+ io.write(cipher.update(data))
+ io.write(cipher.final)
+ io.rewind
+
+ return io
+end
+
+def invidious_companion_encrypt(data)
+ timestamp = Time.utc.to_unix
+ encrypted_data = encrypt_ecb_without_salt("#{timestamp}|#{data}", CONFIG.invidious_companion_key)
+ return Base64.urlsafe_encode(encrypted_data)
end
diff --git a/src/invidious/helpers/webvtt.cr b/src/invidious/helpers/webvtt.cr
new file mode 100644
index 00000000..260d250f
--- /dev/null
+++ b/src/invidious/helpers/webvtt.cr
@@ -0,0 +1,81 @@
+# Namespace for logic relating to generating WebVTT files
+#
+# Probably not compliant to WebVTT's specs but it is enough for Invidious.
+module WebVTT
+ # A WebVTT builder generates WebVTT files
+ private class Builder
+ # See https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#cue_payload
+ private ESCAPE_SUBSTITUTIONS = {
+ '&' => "&",
+ '<' => "<",
+ '>' => ">",
+ '\u200E' => "",
+ '\u200F' => "",
+ '\u00A0' => " ",
+ }
+
+ def initialize(@io : IO)
+ end
+
+ # Writes an vtt cue with the specified time stamp and contents
+ def cue(start_time : Time::Span, end_time : Time::Span, text : String)
+ timestamp(start_time, end_time)
+ @io << self.escape(text)
+ @io << "\n\n"
+ end
+
+ private def timestamp(start_time : Time::Span, end_time : Time::Span)
+ timestamp_component(start_time)
+ @io << " --> "
+ timestamp_component(end_time)
+
+ @io << '\n'
+ end
+
+ private def timestamp_component(timestamp : Time::Span)
+ @io << timestamp.hours.to_s.rjust(2, '0')
+ @io << ':' << timestamp.minutes.to_s.rjust(2, '0')
+ @io << ':' << timestamp.seconds.to_s.rjust(2, '0')
+ @io << '.' << timestamp.milliseconds.to_s.rjust(3, '0')
+ end
+
+ private def escape(text : String) : String
+ return text.gsub(ESCAPE_SUBSTITUTIONS)
+ end
+
+ def document(setting_fields : Hash(String, String)? = nil, &)
+ @io << "WEBVTT\n"
+
+ if setting_fields
+ setting_fields.each do |name, value|
+ @io << name << ": " << value << '\n'
+ end
+ end
+
+ @io << '\n'
+
+ yield
+ end
+ end
+
+ # Returns the resulting `String` of writing WebVTT to the yielded `WebVTT::Builder`
+ #
+ # ```
+ # string = WebVTT.build do |vtt|
+ # vtt.cue(Time::Span.new(seconds: 1), Time::Span.new(seconds: 2), "Line 1")
+ # vtt.cue(Time::Span.new(seconds: 2), Time::Span.new(seconds: 3), "Line 2")
+ # end
+ #
+ # string # => "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nLine 1\n\n00:00:02.000 --> 00:00:03.000\nLine 2\n\n"
+ # ```
+ #
+ # Accepts an optional settings fields hash to add settings attribute to the resulting vtt file.
+ def self.build(setting_fields : Hash(String, String)? = nil, &)
+ String.build do |str|
+ builder = Builder.new(str)
+ builder.document(setting_fields) do
+ yield builder
+ end
+ end
+ end
+end
diff --git a/src/invidious/http_server/utils.cr b/src/invidious/http_server/utils.cr
new file mode 100644
index 00000000..623a9177
--- /dev/null
+++ b/src/invidious/http_server/utils.cr
@@ -0,0 +1,41 @@
+require "uri"
+
+module Invidious::HttpServer
+ module Utils
+ extend self
+
+ def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
+ url = URI.parse(raw_url)
+
+ # Add some URL parameters
+ params = url.query_params
+ params["host"] = url.host.not_nil! # Should never be nil, in theory
+ params["region"] = region if !region.nil?
+ url.query_params = params
+
+ if absolute
+ return "#{HOST_URL}#{url.request_target}"
+ else
+ return url.request_target
+ end
+ end
+
+ def add_params_to_url(url : String | URI, params : URI::Params) : URI
+ url = URI.parse(url) if url.is_a?(String)
+
+ url_query = url.query || ""
+
+ # Append the parameters
+ url.query = String.build do |str|
+ if !url_query.empty?
+ str << url_query
+ str << '&'
+ end
+
+ str << params
+ end
+
+ return url
+ end
+ end
+end
diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr
index ec0cad64..b6b673f7 100644
--- a/src/invidious/jobs.cr
+++ b/src/invidious/jobs.cr
@@ -1,12 +1,39 @@
module Invidious::Jobs
JOBS = [] of BaseJob
+ # Automatically generate a structure that wraps the various
+ # jobs' configs, so that the following YAML config can be used:
+ #
+ # jobs:
+ # job_name:
+ # enabled: true
+ # some_property: "value"
+ #
+ macro finished
+ struct JobsConfig
+ include YAML::Serializable
+
+ {% for sc in BaseJob.subclasses %}
+ # Voodoo macro to transform `Some::Module::CustomJob` to `custom`
+ {% class_name = sc.id.split("::").last.id.gsub(/Job$/, "").underscore %}
+
+ getter {{ class_name }} = {{ sc.name }}::Config.new
+ {% end %}
+
+ def initialize
+ end
+ end
+ end
+
def self.register(job : BaseJob)
JOBS << job
end
def self.start_all
JOBS.each do |job|
+ # Don't run the main rountine if the job is disabled by config
+ next if job.disabled?
+
spawn { job.begin }
end
end
diff --git a/src/invidious/jobs/base_job.cr b/src/invidious/jobs/base_job.cr
index 47e75864..f90f0bfe 100644
--- a/src/invidious/jobs/base_job.cr
+++ b/src/invidious/jobs/base_job.cr
@@ -1,3 +1,33 @@
abstract class Invidious::Jobs::BaseJob
abstract def begin
+
+ # When this base job class is inherited, make sure to define
+ # a basic "Config" structure, that contains the "enable" property,
+ # and to create the associated instance property.
+ #
+ macro inherited
+ macro finished
+ # This config structure can be expanded as required.
+ struct Config
+ include YAML::Serializable
+
+ property enable = true
+
+ def initialize
+ end
+ end
+
+ property cfg = Config.new
+
+ # Return true if job is enabled by config
+ protected def enabled? : Bool
+ return (@cfg.enable == true)
+ end
+
+ # Return true if job is disabled by config
+ protected def disabled? : Bool
+ return (@cfg.enable == false)
+ end
+ end
+ end
end
diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr
deleted file mode 100644
index 71f8a938..00000000
--- a/src/invidious/jobs/bypass_captcha_job.cr
+++ /dev/null
@@ -1,135 +0,0 @@
-class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
- def begin
- loop do
- begin
- random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String})
- if !random_video
- random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"}
- end
- {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path|
- response = YT_POOL.client &.get(path)
- if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
- html = XML.parse_html(response.body)
- form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
- site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
- s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
-
- inputs = {} of String => String
- form.xpath_nodes(%(.//input[@name])).map do |node|
- inputs[node["name"]] = node["value"]
- end
-
- headers = response.cookies.add_request_headers(HTTP::Headers.new)
-
- response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask",
- headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
- "clientKey" => CONFIG.captcha_key,
- "task" => {
- "type" => "NoCaptchaTaskProxyless",
- "websiteURL" => "https://www.youtube.com#{path}",
- "websiteKey" => site_key,
- "recaptchaDataSValue" => s_value,
- },
- }.to_json).body)
-
- raise response["error"].as_s if response["error"]?
- task_id = response["taskId"].as_i
-
- loop do
- sleep 10.seconds
-
- response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult",
- headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
- "clientKey" => CONFIG.captcha_key,
- "taskId" => task_id,
- }.to_json).body)
-
- if response["status"]?.try &.== "ready"
- break
- elsif response["errorId"]?.try &.as_i != 0
- raise response["errorDescription"].as_s
- end
- end
-
- inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
- headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
- response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
-
- response.cookies
- .select { |cookie| cookie.name != "PREF" }
- .each { |cookie| CONFIG.cookies << cookie }
-
- # Persist cookies between runs
- File.write("config/config.yml", CONFIG.to_yaml)
- elsif response.headers["Location"]?.try &.includes?("/sorry/index")
- location = response.headers["Location"].try { |u| URI.parse(u) }
- headers = HTTP::Headers{":authority" => location.host.not_nil!}
- response = YT_POOL.client &.get(location.request_target, headers)
-
- html = XML.parse_html(response.body)
- form = html.xpath_node(%(//form[@action="index"])).not_nil!
- site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
- s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
-
- inputs = {} of String => String
- form.xpath_nodes(%(.//input[@name])).map do |node|
- inputs[node["name"]] = node["value"]
- end
-
- captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url))
- captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
- response = JSON.parse(captcha_client.post("/createTask",
- headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
- "clientKey" => CONFIG.captcha_key,
- "task" => {
- "type" => "NoCaptchaTaskProxyless",
- "websiteURL" => location.to_s,
- "websiteKey" => site_key,
- "recaptchaDataSValue" => s_value,
- },
- }.to_json).body)
-
- captcha_client.close
-
- raise response["error"].as_s if response["error"]?
- task_id = response["taskId"].as_i
-
- loop do
- sleep 10.seconds
-
- response = JSON.parse(captcha_client.post("/getTaskResult",
- headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
- "clientKey" => CONFIG.captcha_key,
- "taskId" => task_id,
- }.to_json).body)
-
- if response["status"]?.try &.== "ready"
- break
- elsif response["errorId"]?.try &.as_i != 0
- raise response["errorDescription"].as_s
- end
- end
-
- inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
- headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
- response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
- headers = HTTP::Headers{
- "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
- }
- cookies = HTTP::Cookies.from_client_headers(headers)
-
- cookies.each { |cookie| CONFIG.cookies << cookie }
-
- # Persist cookies between runs
- File.write("config/config.yml", CONFIG.to_yaml)
- end
- end
- rescue ex
- LOGGER.error("BypassCaptchaJob: #{ex.message}")
- ensure
- sleep 1.minute
- Fiber.yield
- end
- end
- end
-end
diff --git a/src/invidious/jobs/clear_expired_items_job.cr b/src/invidious/jobs/clear_expired_items_job.cr
new file mode 100644
index 00000000..17191aac
--- /dev/null
+++ b/src/invidious/jobs/clear_expired_items_job.cr
@@ -0,0 +1,27 @@
+class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob
+ # Remove items (videos, nonces, etc..) whose cache is outdated every hour.
+ # Removes the need for a cron job.
+ def begin
+ loop do
+ failed = false
+
+ LOGGER.info("jobs: running ClearExpiredItems job")
+
+ begin
+ Invidious::Database::Videos.delete_expired
+ Invidious::Database::Nonces.delete_expired
+ rescue DB::Error
+ failed = true
+ end
+
+ # Retry earlier than scheduled on DB error
+ if failed
+ LOGGER.info("jobs: ClearExpiredItems failed. Retrying in 10 minutes.")
+ sleep 10.minutes
+ else
+ LOGGER.info("jobs: ClearExpiredItems done.")
+ sleep 1.hour
+ end
+ end
+ end
+end
diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr
new file mode 100644
index 00000000..cb4280b9
--- /dev/null
+++ b/src/invidious/jobs/instance_refresh_job.cr
@@ -0,0 +1,97 @@
+class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob
+ # We update the internals of a constant as so it can be accessed from anywhere
+ # within the codebase
+ #
+ # "INSTANCES" => Array(Tuple(String, String)) # region, instance
+
+ INSTANCES = {"INSTANCES" => [] of Tuple(String, String)}
+
+ def initialize
+ end
+
+ def begin
+ loop do
+ refresh_instances
+ LOGGER.info("InstanceListRefreshJob: Done, sleeping for 30 minutes")
+ sleep 30.minute
+ Fiber.yield
+ end
+ end
+
+ # Refreshes the list of instances used for redirects.
+ #
+ # Does the following three checks for each instance
+ # - Is it a clear-net instance?
+ # - Is it an instance with a good uptime?
+ # - Is it an updated instance?
+ private def refresh_instances
+ raw_instance_list = self.fetch_instances
+ filtered_instance_list = [] of Tuple(String, String)
+
+ raw_instance_list.each do |instance_data|
+ # TODO allow Tor hidden service instances when the current instance
+ # is also a hidden service. Same for i2p and any other non-clearnet instances.
+ begin
+ domain = instance_data[0]
+ info = instance_data[1]
+ stats = info["stats"]
+
+ next unless info["type"] == "https"
+ next if bad_uptime?(info["monitor"])
+ next if outdated?(stats["software"]["version"])
+
+ filtered_instance_list << {info["region"].as_s, domain.as_s}
+ rescue ex
+ if domain
+ LOGGER.info("InstanceListRefreshJob: failed to parse information from '#{domain}' because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ")
+ else
+ LOGGER.info("InstanceListRefreshJob: failed to parse information from an instance because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ")
+ end
+ end
+ end
+
+ if !filtered_instance_list.empty?
+ INSTANCES["INSTANCES"] = filtered_instance_list
+ end
+ end
+
+ # Fetches information regarding instances from api.invidious.io or an otherwise configured URL
+ private def fetch_instances : Array(JSON::Any)
+ begin
+ # We directly call the stdlib HTTP::Client here as it allows us to negate the effects
+ # of the force_resolve config option. This is needed as api.invidious.io does not support ipv6
+ # and as such the following request raises if we were to use force_resolve with the ipv6 value.
+ instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io"))
+
+ # Timeouts
+ instance_api_client.connect_timeout = 10.seconds
+ instance_api_client.dns_timeout = 10.seconds
+
+ raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a
+ instance_api_client.close
+ rescue ex : Socket::ConnectError | IO::TimeoutError | JSON::ParseException
+ raw_instance_list = [] of JSON::Any
+ end
+
+ return raw_instance_list
+ end
+
+ # Checks if the given target instance is outdated
+ private def outdated?(target_instance_version) : Bool
+ remote_commit_date = target_instance_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/)
+ return false if !remote_commit_date
+
+ remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC)
+ local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC)
+
+ return (remote_commit_date - local_commit_date).abs.days > 30
+ end
+
+ # Checks if the uptime of the target instance is greater than 90% over a 30 day period
+ private def bad_uptime?(target_instance_health_monitor) : Bool
+ return true if !target_instance_health_monitor["down"].as_bool == false
+ return true if target_instance_health_monitor["uptime"].as_f < 90
+
+ return false
+ end
+end
diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr
index 2f525e08..968ee47f 100644
--- a/src/invidious/jobs/notification_job.cr
+++ b/src/invidious/jobs/notification_job.cr
@@ -1,15 +1,103 @@
+struct VideoNotification
+ getter video_id : String
+ getter channel_id : String
+ getter published : Time
+
+ def_hash @channel_id, @video_id
+
+ def ==(other)
+ video_id == other.video_id
+ end
+
+ def self.from_video(video : ChannelVideo) : self
+ VideoNotification.new(video.id, video.ucid, video.published)
+ end
+
+ def initialize(@video_id, @channel_id, @published)
+ end
+
+ def clone : VideoNotification
+ VideoNotification.new(video_id.clone, channel_id.clone, published.clone)
+ end
+end
+
class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
- private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
+ private getter notification_channel : ::Channel(VideoNotification)
+ private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
private getter pg_url : URI
- def initialize(@connection_channel, @pg_url)
+ def initialize(@notification_channel, @connection_channel, @pg_url)
end
def begin
- connections = [] of Channel(PQ::Notification)
+ connections = [] of ::Channel(PQ::Notification)
PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
+ # hash of channels to their videos (id+published) that need notifying
+ to_notify = Hash(String, Set(VideoNotification)).new(
+ ->(hash : Hash(String, Set(VideoNotification)), key : String) {
+ hash[key] = Set(VideoNotification).new
+ }
+ )
+ notify_mutex = Mutex.new
+
+ # fiber to locally cache all incoming notifications (from pubsub webhooks and refresh channels job)
+ spawn do
+ begin
+ loop do
+ notification = notification_channel.receive
+ notify_mutex.synchronize do
+ to_notify[notification.channel_id] << notification
+ end
+ end
+ end
+ end
+ # fiber to regularly persist all cached notifications
+ spawn do
+ loop do
+ begin
+ LOGGER.debug("NotificationJob: waking up")
+ cloned = {} of String => Set(VideoNotification)
+ notify_mutex.synchronize do
+ cloned = to_notify.clone
+ to_notify.clear
+ end
+
+ cloned.each do |channel_id, notifications|
+ if notifications.empty?
+ next
+ end
+
+ LOGGER.info("NotificationJob: updating channel #{channel_id} with #{notifications.size} notifications")
+ if CONFIG.enable_user_notifications
+ video_ids = notifications.map(&.video_id)
+ Invidious::Database::Users.add_multiple_notifications(channel_id, video_ids)
+ PG_DB.using_connection do |conn|
+ notifications.each do |n|
+ # Deliver notifications to `/api/v1/auth/notifications`
+ payload = {
+ "topic" => n.channel_id,
+ "videoId" => n.video_id,
+ "published" => n.published.to_unix,
+ }.to_json
+ conn.exec("NOTIFY notifications, E'#{payload}'")
+ end
+ end
+ else
+ Invidious::Database::Users.feed_needs_update(channel_id)
+ end
+ end
+
+ LOGGER.trace("NotificationJob: Done, sleeping")
+ rescue ex
+ LOGGER.error("NotificationJob: #{ex.message}")
+ end
+ sleep 1.minute
+ Fiber.yield
+ end
+ end
+
loop do
action, connection = connection_channel.receive
diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr
index 92681408..80812a63 100644
--- a/src/invidious/jobs/refresh_channels_job.cr
+++ b/src/invidious/jobs/refresh_channels_job.cr
@@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
max_fibers = CONFIG.channel_threads
lim_fibers = max_fibers
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
backoff = 2.minutes
loop do
diff --git a/src/invidious/jobs/refresh_feeds_job.cr b/src/invidious/jobs/refresh_feeds_job.cr
index 4b52c959..4f8130df 100644
--- a/src/invidious/jobs/refresh_feeds_job.cr
+++ b/src/invidious/jobs/refresh_feeds_job.cr
@@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
def begin
max_fibers = CONFIG.feed_threads
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
loop do
db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|
diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr
index a113bd77..66c91ad5 100644
--- a/src/invidious/jobs/statistics_refresh_job.cr
+++ b/src/invidious/jobs/statistics_refresh_job.cr
@@ -18,6 +18,13 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => 0_i64,
},
+
+ #
+ # "totalRequests" => 0_i64,
+ # "successfulRequests" => 0_i64
+ # "ratio" => 0_i64
+ #
+ "playback" => {} of String => Int64 | Float64,
}
private getter db : DB::Database
@@ -30,7 +37,7 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
loop do
refresh_stats
- sleep 1.minute
+ sleep 10.minute
Fiber.yield
end
end
@@ -49,12 +56,15 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob
users = STATISTICS.dig("usage", "users").as(Hash(String, Int64))
users["total"] = Invidious::Database::Statistics.count_users_total
- users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m
- users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m
+ users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_6m
+ users["activeMonth"] = Invidious::Database::Statistics.count_users_active_1m
STATISTICS["metadata"] = {
"updatedAt" => Time.utc.to_unix,
"lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64,
}
+
+ # Reset playback requests tracker
+ STATISTICS["playback"] = {} of String => Int64 | Float64
end
end
diff --git a/src/invidious/jobs/subscribe_to_feeds_job.cr b/src/invidious/jobs/subscribe_to_feeds_job.cr
index a431a48a..8584fb9c 100644
--- a/src/invidious/jobs/subscribe_to_feeds_job.cr
+++ b/src/invidious/jobs/subscribe_to_feeds_job.cr
@@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
end
active_fibers = 0
- active_channel = Channel(Bool).new
+ active_channel = ::Channel(Bool).new
loop do
db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|
diff --git a/src/invidious/jobs/update_decrypt_function_job.cr b/src/invidious/jobs/update_decrypt_function_job.cr
deleted file mode 100644
index 6fa0ae1b..00000000
--- a/src/invidious/jobs/update_decrypt_function_job.cr
+++ /dev/null
@@ -1,14 +0,0 @@
-class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
- def begin
- loop do
- begin
- DECRYPT_FUNCTION.update_decrypt_function
- rescue ex
- LOGGER.error("UpdateDecryptFunctionJob : #{ex.message}")
- ensure
- sleep 1.minute
- Fiber.yield
- end
- end
- end
-end
diff --git a/src/invidious/jsonify/api_v1/common.cr b/src/invidious/jsonify/api_v1/common.cr
new file mode 100644
index 00000000..64b06465
--- /dev/null
+++ b/src/invidious/jsonify/api_v1/common.cr
@@ -0,0 +1,18 @@
+require "json"
+
+module Invidious::JSONify::APIv1
+ extend self
+
+ def thumbnails(json : JSON::Builder, id : String)
+ json.array do
+ build_thumbnails(id).each do |thumbnail|
+ json.object do
+ json.field "quality", thumbnail[:name]
+ json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
+ json.field "width", thumbnail[:width]
+ json.field "height", thumbnail[:height]
+ end
+ end
+ end
+ end
+end
diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr
new file mode 100644
index 00000000..58805af2
--- /dev/null
+++ b/src/invidious/jsonify/api_v1/video_json.cr
@@ -0,0 +1,301 @@
+require "json"
+
+module Invidious::JSONify::APIv1
+ extend self
+
+ def video(video : Video, json : JSON::Builder, *, locale : String?, proxy : Bool = false)
+ json.object do
+ json.field "type", video.video_type
+
+ json.field "title", video.title
+ json.field "videoId", video.id
+
+ json.field "error", video.info["reason"] if video.info["reason"]?
+
+ json.field "videoThumbnails" do
+ self.thumbnails(json, video.id)
+ end
+ json.field "storyboards" do
+ self.storyboards(json, video.id, video.storyboards)
+ end
+
+ json.field "description", video.description
+ json.field "descriptionHtml", video.description_html
+ json.field "published", video.published.to_unix
+ json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale))
+ json.field "keywords", video.keywords
+
+ json.field "viewCount", video.views
+ json.field "likeCount", video.likes
+ json.field "dislikeCount", 0_i64
+
+ json.field "paid", video.paid
+ json.field "premium", video.premium
+ json.field "isFamilyFriendly", video.is_family_friendly
+ json.field "allowedRegions", video.allowed_regions
+ json.field "genre", video.genre
+ json.field "genreUrl", video.genre_url
+
+ json.field "author", video.author
+ json.field "authorId", video.ucid
+ json.field "authorUrl", "/channel/#{video.ucid}"
+ json.field "authorVerified", video.author_verified
+
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", video.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+
+ json.field "subCountText", video.sub_count_text
+
+ json.field "lengthSeconds", video.length_seconds
+ json.field "allowRatings", video.allow_ratings
+ json.field "rating", 0_i64
+ json.field "isListed", video.is_listed
+ json.field "liveNow", video.live_now
+ json.field "isPostLiveDvr", video.post_live_dvr
+ json.field "isUpcoming", video.upcoming?
+
+ if video.premiere_timestamp
+ json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix
+ end
+
+ if hlsvp = video.hls_manifest_url
+ hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
+ json.field "hlsUrl", hlsvp
+ end
+
+ json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{video.id}"
+
+ json.field "adaptiveFormats" do
+ json.array do
+ video.adaptive_fmts.each do |fmt|
+ json.object do
+ # Only available on regular videos, not livestreams/OTF streams
+ if init_range = fmt["initRange"]?
+ json.field "init", "#{init_range["start"]}-#{init_range["end"]}"
+ end
+ if index_range = fmt["indexRange"]?
+ json.field "index", "#{index_range["start"]}-#{index_range["end"]}"
+ end
+
+ # Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only)
+ json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
+
+ if proxy
+ json.field "url", Invidious::HttpServer::Utils.proxy_video_url(
+ fmt["url"].to_s, absolute: true
+ )
+ else
+ json.field "url", fmt["url"]
+ end
+
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
+ json.field "clen", fmt["contentLength"]? || "-1"
+
+ # Last modified is a unix timestamp with µS, with the dot omitted.
+ # E.g: 1638056732(.)141582
+ #
+ # On livestreams, it's not present, so always fall back to the
+ # current unix timestamp (up to mS precision) for compatibility.
+ last_modified = fmt["lastModified"]?
+ last_modified ||= "#{Time.utc.to_unix_ms}000"
+ json.field "lmt", last_modified
+
+ json.field "projectionType", fmt["projectionType"]
+
+ height = fmt["height"]?.try &.as_i
+ width = fmt["width"]?.try &.as_i
+
+ fps = fmt["fps"]?.try &.as_i
+
+ if fps
+ json.field "fps", fps
+ end
+
+ if height && width
+ json.field "size", "#{width}x#{height}"
+ json.field "resolution", "#{height}p"
+
+ quality_label = "#{width > height ? height : width}p"
+
+ if fps && fps > 30
+ quality_label += fps.to_s
+ end
+
+ json.field "qualityLabel", quality_label
+ end
+
+ if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
+ json.field "container", fmt_info["ext"]
+ json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
+ end
+
+ # Livestream chunk infos
+ json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec")
+ json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec")
+
+ # Audio-related data
+ json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality")
+ json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate")
+ json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels")
+
+ # Extra misc stuff
+ json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo")
+ json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack")
+ end
+ end
+ end
+ end
+
+ json.field "formatStreams" do
+ json.array do
+ video.fmt_stream.each do |fmt|
+ json.object do
+ if proxy
+ json.field "url", Invidious::HttpServer::Utils.proxy_video_url(
+ fmt["url"].to_s, absolute: true
+ )
+ else
+ json.field "url", fmt["url"]
+ end
+ json.field "itag", fmt["itag"].as_i.to_s
+ json.field "type", fmt["mimeType"]
+ json.field "quality", fmt["quality"]
+
+ json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
+
+ height = fmt["height"]?.try &.as_i
+ width = fmt["width"]?.try &.as_i
+
+ fps = fmt["fps"]?.try &.as_i
+
+ if fps
+ json.field "fps", fps
+ end
+
+ if height && width
+ json.field "size", "#{width}x#{height}"
+ json.field "resolution", "#{height}p"
+
+ quality_label = "#{width > height ? height : width}p"
+
+ if fps && fps > 30
+ quality_label += fps.to_s
+ end
+
+ json.field "qualityLabel", quality_label
+ end
+
+ if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
+ json.field "container", fmt_info["ext"]
+ json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
+ end
+ end
+ end
+ end
+ end
+
+ json.field "captions" do
+ json.array do
+ video.captions.each do |caption|
+ json.object do
+ json.field "label", caption.name
+ json.field "language_code", caption.language_code
+ json.field "url", "/api/v1/captions/#{video.id}?label=#{URI.encode_www_form(caption.name)}"
+ end
+ end
+ end
+ end
+
+ if !video.music.empty?
+ json.field "musicTracks" do
+ json.array do
+ video.music.each do |music|
+ json.object do
+ json.field "song", music.song
+ json.field "artist", music.artist
+ json.field "album", music.album
+ json.field "license", music.license
+ end
+ end
+ end
+ end
+ end
+
+ json.field "recommendedVideos" do
+ json.array do
+ video.related_videos.each do |rv|
+ if rv["id"]?
+ json.object do
+ json.field "videoId", rv["id"]
+ json.field "title", rv["title"]
+ json.field "videoThumbnails" do
+ self.thumbnails(json, rv["id"])
+ end
+
+ json.field "author", rv["author"]
+ json.field "authorUrl", "/channel/#{rv["ucid"]?}"
+ json.field "authorId", rv["ucid"]?
+ json.field "authorVerified", rv["author_verified"] == "true"
+ if rv["author_thumbnail"]?
+ json.field "authorThumbnails" do
+ json.array do
+ qualities = {32, 48, 76, 100, 176, 512}
+
+ qualities.each do |quality|
+ json.object do
+ json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
+ json.field "width", quality
+ json.field "height", quality
+ end
+ end
+ end
+ end
+ end
+
+ json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
+ json.field "viewCountText", rv["short_view_count"]?
+ json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
+ json.field "published", rv["published"]?
+ if rv["published"]?.try &.presence
+ json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale))
+ else
+ json.field "publishedText", ""
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ def storyboards(json, id, storyboards)
+ json.array do
+ storyboards.each do |sb|
+ json.object do
+ json.field "url", "/api/v1/storyboards/#{id}?width=#{sb.width}&height=#{sb.height}"
+ json.field "templateUrl", sb.url.to_s
+ json.field "width", sb.width
+ json.field "height", sb.height
+ json.field "count", sb.count
+ json.field "interval", sb.interval
+ json.field "storyboardWidth", sb.columns
+ json.field "storyboardHeight", sb.rows
+ json.field "storyboardCount", sb.images_count
+ end
+ end
+ end
+ end
+end
diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr
index 3f342b92..28ff0ff6 100644
--- a/src/invidious/mixes.cr
+++ b/src/invidious/mixes.cr
@@ -81,7 +81,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
})
end
-def template_mix(mix)
+def template_mix(mix, listen)
html = <<-END_HTML
@@ -95,9 +95,9 @@ def template_mix(mix)
mix["videos"].as_a.each do |video|
html += <<-END_HTML
+
+ END_HTML
+ end
+ end
+ end
+ end
+end
diff --git a/src/invidious/frontend/comments_youtube.cr b/src/invidious/frontend/comments_youtube.cr
new file mode 100644
index 00000000..a0e1d783
--- /dev/null
+++ b/src/invidious/frontend/comments_youtube.cr
@@ -0,0 +1,208 @@
+module Invidious::Frontend::Comments
+ extend self
+
+ def template_youtube(comments, locale, thin_mode, is_replies = false)
+ String.build do |html|
+ root = comments["comments"].as_a
+ root.each do |child|
+ if child["replies"]?
+ replies_count_text = translate_count(locale,
+ "comments_view_x_replies",
+ child["replies"]["replyCount"].as_i64 || 0,
+ NumberFormatting::Separator
+ )
+
+ replies_html = <<-END_HTML
+
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
+ + [ − ] + #{child.author} + #{translate_count(locale, "comments_points_count", child.score, NumberFormatting::Separator)} + #{translate(locale, "`x` ago", recode_date(child.created_utc, locale))} + #{translate(locale, "permalink")} +
+
+ #{body_html}
+ #{replies_html}
+
+
+
+
+ END_HTML
+ elsif comments["authorId"]? && !comments["singlePost"]?
+ # for posts we should display a link to the post
+ replies_count_text = translate_count(locale,
+ "comments_view_x_replies",
+ child["replyCount"].as_i64 || 0,
+ NumberFormatting::Separator
+ )
+
+ replies_html = <<-END_HTML
+
+
+
+
+
+
+ END_HTML
+ end
+
+ if !thin_mode
+ author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).request_target}"
+ else
+ author_thumbnail = ""
+ end
+
+ author_name = HTML.escape(child["author"].as_s)
+ sponsor_icon = ""
+ if child["verified"]?.try &.as_bool && child["authorIsChannelOwner"]?.try &.as_bool
+ author_name += " "
+ elsif child["verified"]?.try &.as_bool
+ author_name += " "
+ end
+
+ if child["isSponsor"]?.try &.as_bool
+ sponsor_icon = String.build do |str|
+ str << %(
+
+
+
+
+ END_HTML
+ end
+
+ if comments["continuation"]?
+ html << <<-END_HTML
+
+
+
+
+
+ #{translate(locale, "carousel_skip")}
+
+ END_HTML
+ else nil # Ignore
+ end
+ end
+
+ html << <<-END_HTML
+
+ + + #{author_name} + + #{sponsor_icon} +
#{child["contentHtml"]}
+ END_HTML + + if child["attachment"]? + attachment = child["attachment"] + + case attachment["type"] + when "image" + attachment = attachment["imageThumbnails"][1] + + html << <<-END_HTML +
+
+ END_HTML
+ when "video"
+ if attachment["error"]?
+ html << <<-END_HTML
+
+
+
+
+
+
+ END_HTML
+ else
+ html << <<-END_HTML
+ #{attachment["error"]}
+
+
+
+ END_HTML
+ end
+ when "multiImage"
+ html << <<-END_HTML
+
+ END_HTML
+ image_array = attachment["images"].as_a
+
+ image_array.each_index do |i|
+ html << <<-END_HTML
+
+ (i + 1).to_s, "total" => image_array.size.to_s})}" tabindex="0">
+ 
+
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
+
+ END_HTML
+ attachment["images"].as_a.each_index do |i|
+ html << <<-END_HTML
+
+ END_HTML
+ end
+ html << <<-END_HTML
+
+
+
+ #{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}
+ |
+ END_HTML
+
+ if comments["videoId"]?
+ html << <<-END_HTML
+ [YT]
+ |
+ END_HTML
+ elsif comments["authorId"]?
+ html << <<-END_HTML
+ [YT]
+ |
+ END_HTML
+ end
+
+ html << <<-END_HTML
+ #{number_with_separator(child["likeCount"])}
+ END_HTML
+
+ if child["creatorHeart"]?
+ if !thin_mode
+ creator_thumbnail = "/ggpht#{URI.parse(child["creatorHeart"]["creatorThumbnail"].as_s).request_target}"
+ else
+ creator_thumbnail = ""
+ end
+
+ html << <<-END_HTML
+
+
+
+
+
+
+
+
+
+ END_HTML
+ end
+
+ html << <<-END_HTML
+
+
+
+ END_HTML
+ end
+ end
+ end
+end
diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr
new file mode 100644
index 00000000..7a6cf79d
--- /dev/null
+++ b/src/invidious/frontend/misc.cr
@@ -0,0 +1,14 @@
+module Invidious::Frontend::Misc
+ extend self
+
+ def redirect_url(env : HTTP::Server::Context)
+ prefs = env.get("preferences").as(Preferences)
+
+ if prefs.automatic_instance_redirect
+ current_page = env.get?("current_page").as(String)
+ return "/redirect?referer=#{current_page}"
+ else
+ return "https://redirect.invidious.io#{env.request.resource}"
+ end
+ end
+end
diff --git a/src/invidious/frontend/pagination.cr b/src/invidious/frontend/pagination.cr
new file mode 100644
index 00000000..a29f5936
--- /dev/null
+++ b/src/invidious/frontend/pagination.cr
@@ -0,0 +1,121 @@
+require "uri"
+
+module Invidious::Frontend::Pagination
+ extend self
+
+ private def first_page(str : String::Builder, locale : String?, url : String)
+ str << %()
+
+ if locale_is_rtl?(locale)
+ # Inverted arrow ("first" points to the right)
+ str << translate(locale, "First page")
+ str << " "
+ str << %()
+ else
+ # Regular arrow ("first" points to the left)
+ str << %()
+ str << " "
+ str << translate(locale, "First page")
+ end
+
+ str << ""
+ end
+
+ private def previous_page(str : String::Builder, locale : String?, url : String)
+ # Link
+ str << %()
+
+ if locale_is_rtl?(locale)
+ # Inverted arrow ("previous" points to the right)
+ str << translate(locale, "Previous page")
+ str << " "
+ str << %()
+ else
+ # Regular arrow ("previous" points to the left)
+ str << %()
+ str << " "
+ str << translate(locale, "Previous page")
+ end
+
+ str << ""
+ end
+
+ private def next_page(str : String::Builder, locale : String?, url : String)
+ # Link
+ str << %()
+
+ if locale_is_rtl?(locale)
+ # Inverted arrow ("next" points to the left)
+ str << %()
+ str << " "
+ str << translate(locale, "Next page")
+ else
+ # Regular arrow ("next" points to the right)
+ str << translate(locale, "Next page")
+ str << " "
+ str << %()
+ end
+
+ str << ""
+ end
+
+ def nav_numeric(locale : String?, *, base_url : String | URI, current_page : Int, show_next : Bool = true)
+ return String.build do |str|
+ str << %(\n)
+ str << %(
\n\n)
+ end
+ end
+
+ def nav_ctoken(locale : String?, *, base_url : String | URI, ctoken : String?, first_page : Bool, params : URI::Params)
+ return String.build do |str|
+ str << %(\n)
+
+ str << %(
\n)
+ str << %()
+
+ if current_page > 1
+ params_prev = URI::Params{"page" => (current_page - 1).to_s}
+ url_prev = HttpServer::Utils.add_params_to_url(base_url, params_prev)
+
+ self.previous_page(str, locale, url_prev.to_s)
+ end
+
+ str << %(
\n)
+ str << %(\n)
+
+ str << %(\n)
+ str << %(
\n\n)
+ end
+ end
+end
diff --git a/src/invidious/frontend/search_filters.cr b/src/invidious/frontend/search_filters.cr
new file mode 100644
index 00000000..8ac0af2e
--- /dev/null
+++ b/src/invidious/frontend/search_filters.cr
@@ -0,0 +1,135 @@
+module Invidious::Frontend::SearchFilters
+ extend self
+
+ # Generate the search filters collapsable widget.
+ def generate(filters : Search::Filters, query : String, page : Int, locale : String) : String
+ return String.build(8000) do |str|
+ str << "\n)
+
+ str << %(
\n)
+ str << %()
+
+ if !first_page
+ self.first_page(str, locale, base_url.to_s)
+ end
+
+ str << %(
\n)
+
+ str << %(\n)
+
+ str << %(\n"
+ str << "\t
\n"
+ end
+ end
+
+ # Generate wrapper HTML (`"
+ str << "\t\t
\n"
+ str << "" << translate(locale, "search_filters_title") << "
\n" + + str << "\t\t\n" + + str << "\t`, filter name, etc...) around the
+ # `` elements of a search filter
+ macro filter_wrapper(name)
+ str << "\t\t\t\t\n"
+ end
+
+ # Generates the HTML for the list of radio buttons of the "date" search filter
+ def make_date_filter_options(str : String::Builder, value : Search::Filters::Date, locale : String)
+ {% for value in Invidious::Search::Filters::Date.constants %}
+ {% date = value.underscore %}
+
+ str << "\t\t\t\t\t\t"
@@ -42,15 +48,14 @@ module Invidious::Frontend::WatchPage
str << translate(locale, "Download as: ")
str << "\n"
- # TODO: remove inline style
- str << "\t\t
END_HTML
@@ -103,7 +106,7 @@ end
# -------------------
macro error_atom(*args)
- error_atom_helper(env, {{*args}})
+ error_atom_helper(env, {{args.splat}})
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
@@ -129,14 +132,14 @@ end
# -------------------
macro error_json(*args)
- error_json_helper(env, {{*args}})
+ error_json_helper(env, {{args.splat}})
end
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
exception : Exception,
- additional_fields : Hash(String, Object) | Nil = nil
+ additional_fields : Hash(String, Object) | Nil = nil,
)
if exception.is_a?(InfoException)
return error_json_helper(env, status_code, exception.message || "", additional_fields)
@@ -158,7 +161,7 @@ def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
message : String,
- additional_fields : Hash(String, Object) | Nil = nil
+ additional_fields : Hash(String, Object) | Nil = nil,
)
env.response.content_type = "application/json"
env.response.status_code = status_code
@@ -198,7 +201,7 @@ def error_redirect_helper(env : HTTP::Server::Context)
#{switch_instance}
"
+ str << "'
+
+ str << "
\n"
+ {% end %}
+ end
+
+ # Generates the HTML for the list of radio buttons of the "type" search filter
+ def make_type_filter_options(str : String::Builder, value : Search::Filters::Type, locale : String)
+ {% for value in Invidious::Search::Filters::Type.constants %}
+ {% type = value.underscore %}
+
+ str << "\t\t\t\t\t\t"
+ str << "'
+
+ str << "
\n"
+ {% end %}
+ end
+
+ # Generates the HTML for the list of radio buttons of the "duration" search filter
+ def make_duration_filter_options(str : String::Builder, value : Search::Filters::Duration, locale : String)
+ {% for value in Invidious::Search::Filters::Duration.constants %}
+ {% duration = value.underscore %}
+
+ str << "\t\t\t\t\t\t"
+ str << "'
+
+ str << "
\n"
+ {% end %}
+ end
+
+ # Generates the HTML for the list of checkboxes of the "features" search filter
+ def make_features_filter_options(str : String::Builder, value : Search::Filters::Features, locale : String)
+ {% for value in Invidious::Search::Filters::Features.constants %}
+ {% if value.stringify != "All" && value.stringify != "None" %}
+ {% feature = value.underscore %}
+
+ str << "\t\t\t\t\t\t"
+ str << "'
+
+ str << "
\n"
+ {% end %}
+ {% end %}
+ end
+
+ # Generates the HTML for the list of radio buttons of the "sort" search filter
+ def make_sort_filter_options(str : String::Builder, value : Search::Filters::Sort, locale : String)
+ {% for value in Invidious::Search::Filters::Sort.constants %}
+ {% sort = value.underscore %}
+
+ str << "\t\t\t\t\t\t"
+ str << "'
+
+ str << "
\n"
+ {% end %}
+ end
+end
diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr
index 80b67641..15d925e3 100644
--- a/src/invidious/frontend/watch_page.cr
+++ b/src/invidious/frontend/watch_page.cr
@@ -7,26 +7,32 @@ module Invidious::Frontend::WatchPage
getter full_videos : Array(Hash(String, JSON::Any))
getter video_streams : Array(Hash(String, JSON::Any))
getter audio_streams : Array(Hash(String, JSON::Any))
- getter captions : Array(Caption)
+ getter captions : Array(Invidious::Videos::Captions::Metadata)
def initialize(
@full_videos,
@video_streams,
@audio_streams,
- @captions
+ @captions,
)
end
end
def download_widget(locale : String, video : Video, video_assets : VideoAssets) : String
if CONFIG.disabled?("downloads")
- return "#{translate(locale, "Download is disabled.")}
" + return "#{translate(locale, "Download is disabled")}
" + end + + url = "/download" + if (CONFIG.invidious_companion.present?) + invidious_companion = CONFIG.invidious_companion.sample + url = "#{invidious_companion.public_url}/download?check=#{invidious_companion_encrypt(video.id)}" end return String.build(4000) do |str| str << ")|(
": "\n", @@ -103,15 +78,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = published response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -138,15 +104,6 @@ def create_notification_stream(env, topics, connection_channel) Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| response = JSON.parse(video.to_json(locale)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -180,15 +137,6 @@ def create_notification_stream(env, topics, connection_channel) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) - if fields_text = env.params.query["fields"]? - begin - JSONFilter.filter(response, fields_text) - rescue ex - env.response.status_code = 400 - response = {"error" => ex.message} - end - end - env.response.puts "id: #{id}" env.response.puts "data: #{response.to_json}" env.response.puts @@ -233,3 +181,20 @@ def proxy_file(response, env) IO.copy response.body_io, env.response end end + +# Fetch the playback requests tracker from the statistics endpoint. +# +# Creates a new tracker when unavailable. +def get_playback_statistic + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? + tracker = { + "totalRequests" => 0_i64, + "successfulRequests" => 0_i64, + "ratio" => 0_f64, + } + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker + end + + return tracker.as(Hash(String, Int64 | Float64)) +end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 39e183f2..bca2edda 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,19 +1,35 @@ -# "bn_BD" => load_locale("bn_BD"), # Bengali (Bangladesh) [Incomplete] -# "eu" => load_locale("eu"), # Basque [Incomplete] -# "sk" => load_locale("sk"), # Slovak [Incomplete] +# Languages requiring a better level of translation (at least 20%) +# to be added to the list below: +# +# "af" => "", # Afrikaans +# "az" => "", # Azerbaijani +# "be" => "", # Belarusian +# "bn_BD" => "", # Bengali (Bangladesh) +# "ia" => "", # Interlingua +# "or" => "", # Odia +# "tk" => "", # Turkmen +# "tok => "", # Toki Pona +# LOCALES_LIST = { "ar" => "العربية", # Arabic + "bg" => "български", # Bulgarian + "bn" => "বাংলা", # Bengali + "ca" => "Català", # Catalan "cs" => "Čeština", # Czech + "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek "en-US" => "English", # English "eo" => "Esperanto", # Esperanto "es" => "Español", # Spanish + "et" => "Eesti keel", # Estonian + "eu" => "Euskara", # Basque "fa" => "فارسی", # Persian "fi" => "Suomi", # Finnish "fr" => "Français", # French "he" => "עברית", # Hebrew + "hi" => "हिन्दी", # Hindi "hr" => "Hrvatski", # Croatian "hu-HU" => "Magyar Nyelv", # Hungarian "id" => "Bahasa Indonesia", # Indonesian @@ -21,6 +37,7 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean + "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch @@ -29,11 +46,15 @@ LOCALES_LIST = { "pt-BR" => "Português Brasileiro", # Portuguese (Brazil) "pt-PT" => "Português de Portugal", # Portuguese (Portugal) "ro" => "Română", # Romanian - "ru" => "русский", # Russian + "ru" => "Русский", # Russian + "si" => "සිංහල", # Sinhala + "sk" => "Slovenčina", # Slovak + "sl" => "Slovenščina", # Slovenian "sq" => "Shqip", # Albanian - "sr" => "srpski (latinica)", # Serbian (Latin) - "sr_Cyrl" => "српски (ћирилица)", # Serbian (Cyrillic) + "sr" => "Srpski (latinica)", # Serbian (Latin) + "sr_Cyrl" => "Српски (ћирилица)", # Serbian (Cyrillic) "sv-SE" => "Svenska", # Swedish + "ta" => "தமிழ்", # Tamil "tr" => "Türkçe", # Turkish "uk" => "Українська", # Ukrainian "vi" => "Tiếng Việt", # Vietnamese @@ -73,7 +94,7 @@ def load_all_locales return locales end -def translate(locale : String?, key : String, text : String | Nil = nil) : String +def translate(locale : String?, key : String, text : String | Hash(String, String) | Nil = nil) : String # Log a warning if "key" doesn't exist in en-US locale and return # that key as the text, so this is more or less transparent to the user. if !LOCALES["en-US"].has_key?(key) @@ -96,10 +117,12 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin match_length = 0 raw_data.as_h.each do |hash_key, value| - if md = text.try &.match(/#{hash_key}/) - if md[0].size >= match_length - translation = value.as_s - match_length = md[0].size + if text.is_a?(String) + if md = text.try &.match(/#{hash_key}/) + if md[0].size >= match_length + translation = value.as_s + match_length = md[0].size + end end end end @@ -109,8 +132,13 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin raise "Invalid translation \"#{raw_data}\"" end - if text + if text.is_a?(String) translation = translation.gsub("`x`", text) + elsif text.is_a?(Hash(String, String)) + # adds support for multi string interpolation. Based on i18next https://www.i18next.com/translation-function/interpolation#basic + text.each_key do |hash_key| + translation = translation.gsub("{{#{hash_key}}}", text[hash_key]) + end end return translation @@ -160,3 +188,12 @@ def translate_bool(locale : String?, translation : Bool) return translate(locale, "No") end end + +def locale_is_rtl?(locale : String?) + # Fallback to en-US + return false if locale.nil? + + # Arabic, Persian, Hebrew + # See https://en.wikipedia.org/wiki/Right-to-left_script#List_of_RTL_scripts + return {"ar", "fa", "he"}.includes? locale +end diff --git a/src/invidious/helpers/i18next.cr b/src/invidious/helpers/i18next.cr index e84f88fb..684e6d14 100644 --- a/src/invidious/helpers/i18next.cr +++ b/src/invidious/helpers/i18next.cr @@ -35,27 +35,35 @@ module I18next::Plurals Special_Slovenian = 21 Special_Hebrew = 22 Special_Odia = 23 + + # Mixed v3/v4 rules in Weblate + # `es`, `pt` and `pt-PT` doesn't seem to have been refreshed + # by weblate yet, but I suspect it will happen one day. + # See: https://github.com/translate/translate/issues/4873 + Special_French_Portuguese + Special_Hungarian_Serbian + Special_Spanish_Italian end private PLURAL_SETS = { PluralForms::Single_gt_one => [ - "ach", "ak", "am", "arn", "br", "fil", "fr", "gun", "ln", "mfe", "mg", - "mi", "oc", "pt", "pt-BR", "tg", "tl", "ti", "tr", "uz", "wa", + "ach", "ak", "am", "arn", "br", "fa", "fil", "gun", "ln", "mfe", "mg", + "mi", "oc", "pt-PT", "tg", "tl", "ti", "tr", "uz", "wa", ], PluralForms::Single_not_one => [ "af", "an", "ast", "az", "bg", "bn", "ca", "da", "de", "dev", "el", "en", - "eo", "es", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", - "hu", "hy", "ia", "it", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", + "eo", "et", "eu", "fi", "fo", "fur", "fy", "gl", "gu", "ha", "hi", + "hu", "hy", "ia", "kk", "kn", "ku", "lb", "mai", "ml", "mn", "mr", "nah", "nap", "nb", "ne", "nl", "nn", "no", "nso", "pa", "pap", "pms", - "ps", "pt-PT", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", + "ps", "rm", "sco", "se", "si", "so", "son", "sq", "sv", "sw", "ta", "te", "tk", "ur", "yo", ], PluralForms::None => [ - "ay", "bo", "cgg", "fa", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", + "ay", "bo", "cgg", "ht", "id", "ja", "jbo", "ka", "km", "ko", "ky", "lo", "ms", "sah", "su", "th", "tt", "ug", "vi", "wo", "zh", ], PluralForms::Dual_Slavic => [ - "be", "bs", "cnr", "dz", "hr", "ru", "sr", "uk", + "be", "bs", "cnr", "dz", "ru", "uk", ], } @@ -81,6 +89,13 @@ module I18next::Plurals "ro" => PluralForms::Special_Romanian, "sk" => PluralForms::Special_Czech_Slovak, "sl" => PluralForms::Special_Slovenian, + # Mixed v3/v4 rules + "es" => PluralForms::Special_Spanish_Italian, + "fr" => PluralForms::Special_French_Portuguese, + "hr" => PluralForms::Special_Hungarian_Serbian, + "it" => PluralForms::Special_Spanish_Italian, + "pt" => PluralForms::Special_French_Portuguese, + "sr" => PluralForms::Special_Hungarian_Serbian, } # These are the v1 and v2 compatible suffixes. @@ -150,9 +165,8 @@ module I18next::Plurals end def get_plural_form(locale : String) : PluralForms - # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code, - # except for pt-BR and pt-PT which needs to be kept as-is. - if !locale.matches?(/^pt-(BR|PT)$/) + # Extract the ISO 639-1 or 639-2 code from an RFC 5646 language code + if !locale.matches?(/^pt-PT$/) locale = locale.split('-')[0] end @@ -174,7 +188,7 @@ module I18next::Plurals # Emulate the `rule.numbers.size == 2 && rule.numbers[0] == 1` check # from original i18next code - private def is_simple_plural(form : PluralForms) : Bool + private def simple_plural?(form : PluralForms) : Bool case form when .single_gt_one? then return true when .single_not_one? then return true @@ -196,7 +210,7 @@ module I18next::Plurals idx = SuffixIndex.get_index(plural_form, count) # Simple plurals are handled differently in all versions (but v4) - if @simplify_plural_suffix && is_simple_plural(plural_form) + if @simplify_plural_suffix && simple_plural?(plural_form) return (idx == 1) ? "_plural" : "" end @@ -246,6 +260,10 @@ module I18next::Plurals when .special_slovenian? then return special_slovenian(count) when .special_hebrew? then return special_hebrew(count) when .special_odia? then return special_odia(count) + # Mixed v3/v4 forms + when .special_spanish_italian? then return special_cldr_spanish_italian(count) + when .special_french_portuguese? then return special_cldr_french_portuguese(count) + when .special_hungarian_serbian? then return special_cldr_hungarian_serbian(count) else # default, if nothing matched above return 0_u8 @@ -507,5 +525,42 @@ module I18next::Plurals def self.special_odia(count : Int) : UInt8 return (count == 1) ? 0_u8 : 1_u8 end + + # ------------------- + # "v3.5" rules + # ------------------- + + # Plural form for Spanish & Italian languages + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_spanish_italian(count : Int) : UInt8 + return 0_u8 if (count == 1) # one + return 1_u8 if (count != 0 && count % 1_000_000 == 0) # many + return 2_u8 # other + end + + # Plural form for French and Portuguese + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_french_portuguese(count : Int) : UInt8 + return 0_u8 if (count == 0 || count == 1) # one + return 1_u8 if (count % 1_000_000 == 0) # many + return 2_u8 # other + end + + # Plural form for Hungarian and Serbian + # + # This rule is mostly compliant to CLDR v42 + # + def self.special_cldr_hungarian_serbian(count : Int) : UInt8 + n_mod_10 = count % 10 + n_mod_100 = count % 100 + + return 0_u8 if (n_mod_10 == 1 && n_mod_100 != 11) # one + return 1_u8 if (2 <= n_mod_10 <= 4 && (n_mod_100 < 12 || 14 < n_mod_100)) # few + return 2_u8 # other + end end end diff --git a/src/invidious/helpers/json_filter.cr b/src/invidious/helpers/json_filter.cr deleted file mode 100644 index b8e8f96d..00000000 --- a/src/invidious/helpers/json_filter.cr +++ /dev/null @@ -1,248 +0,0 @@ -module JSONFilter - alias BracketIndex = Hash(Int64, Int64) - - alias GroupedFieldsValue = String | Array(GroupedFieldsValue) - alias GroupedFieldsList = Array(GroupedFieldsValue) - - class FieldsParser - class ParseError < Exception - end - - # Returns the `Regex` pattern used to match nest groups - def self.nest_group_pattern : Regex - # uses a '.' character to match json keys as they are allowed - # to contain any unicode codepoint - /(?:|,)(?
@@ -95,9 +95,9 @@ def template_mix(mix)
mix["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
- 
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr
index aefa34cc..7c584d15 100644
--- a/src/invidious/playlists.cr
+++ b/src/invidious/playlists.cr
@@ -46,8 +46,14 @@ struct PlaylistVideo
XML.build { |xml| to_xml(xml) }
end
+ def to_json(locale : String?, json : JSON::Builder)
+ to_json(json)
+ end
+
def to_json(json : JSON::Builder, index : Int32? = nil)
json.object do
+ json.field "type", "video"
+
json.field "title", self.title
json.field "videoId", self.id
@@ -56,7 +62,7 @@ struct PlaylistVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
- generate_thumbnails(json, self.id)
+ Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
if index
@@ -67,6 +73,7 @@ struct PlaylistVideo
end
json.field "lengthSeconds", self.length_seconds
+ json.field "liveNow", self.live_now
end
end
@@ -89,6 +96,7 @@ struct Playlist
property views : Int64
property updated : Time
property thumbnail : String?
+ property subtitle : String?
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
@@ -100,6 +108,7 @@ struct Playlist
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
+ json.field "subtitle", self.subtitle
json.field "authorThumbnails" do
json.array do
@@ -261,7 +270,7 @@ end
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
- title: playlist.title.byte_slice(0, 150),
+ title: playlist.title[..150],
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
@@ -317,7 +326,7 @@ def get_playlist(plid : String)
if playlist = Invidious::Database::Playlists.select(id: plid)
return playlist
else
- raise InfoException.new("Playlist does not exist.")
+ raise NotFoundException.new("Playlist does not exist.")
end
else
return fetch_playlist(plid)
@@ -356,12 +365,16 @@ def fetch_playlist(plid : String)
updated = Time.utc
video_count = 0
+ subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle"))
+
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
+ elsif text.includes? "episode"
+ video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
else
@@ -397,6 +410,7 @@ def fetch_playlist(plid : String)
views: views,
updated: updated,
thumbnail: thumbnail,
+ subtitle: subtitle,
})
end
@@ -418,7 +432,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32,
offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
end
- videos = [] of PlaylistVideo
+ videos = [] of PlaylistVideo | ProblematicTimelineItem
until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
# 100 videos per request
@@ -434,7 +448,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32,
end
def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
- videos = [] of PlaylistVideo
+ videos = [] of PlaylistVideo | ProblematicTimelineItem
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
@@ -486,12 +500,14 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
index: index,
})
end
+ rescue ex
+ videos << ProblematicTimelineItem.new(parse_exception: ex)
end
return videos
end
-def template_playlist(playlist)
+def template_playlist(playlist, listen)
html = <<-END_HTML
@@ -505,9 +521,9 @@ def template_playlist(playlist)
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr
index 9bb73136..c8db207c 100644
--- a/src/invidious/routes/account.cr
+++ b/src/invidious/routes/account.cr
@@ -42,11 +42,6 @@ module Invidious::Routes::Account
sid = sid.as(String)
token = env.params.body["csrf_token"]?
- # We don't store passwords for Google accounts
- if !user.password
- return error_template(400, "Cannot change password for Google accounts")
- end
-
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
@@ -54,11 +49,11 @@ module Invidious::Routes::Account
end
password = env.params.body["password"]?
- if !password
+ if password.nil? || password.empty?
return error_template(401, "Password is a required field")
end
- new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v }
+ new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match")
@@ -203,7 +198,7 @@ module Invidious::Routes::Account
referer = get_referer(env)
if !user
- return env.redirect referer
+ return env.redirect "/login?referer=#{URI.encode_path_segment(env.request.resource)}"
end
user = user.as(User)
@@ -245,7 +240,7 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v }
+ scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
@@ -262,6 +257,7 @@ module Invidious::Routes::Account
end
query["token"] = access_token
+ query["username"] = URI.encode_path_segment(user.email)
url.query = query.to_s
env.redirect url.to_s
@@ -332,17 +328,9 @@ module Invidious::Routes::Account
end
end
- if env.params.query["action_revoke_token"]?
- action = "action_revoke_token"
- else
- return env.redirect referer
- end
-
- session = env.params.query["session"]?
- session ||= ""
-
- case action
- when .starts_with? "action_revoke_token"
+ case action = env.params.query["action"]?
+ when "revoke_token"
+ session = env.params.query["session"]
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
return error_json(400, "Unsupported action #{action}")
diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr
index ca429df5..c27caad7 100644
--- a/src/invidious/routes/api/manifest.cr
+++ b/src/invidious/routes/api/manifest.cr
@@ -8,45 +8,49 @@ module Invidious::Routes::API::Manifest
id = env.params.url["id"]
region = env.params.query["region"]?
+ if CONFIG.invidious_companion.present?
+ invidious_companion = CONFIG.invidious_companion.sample
+ return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
+ end
+
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin
video = get_video(id, region: region)
- rescue ex : VideoRedirect
- return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex : NotFoundException
+ haltf env, status_code: 404
rescue ex
haltf env, status_code: 403
end
if dashmpd = video.dash_manifest_url
- manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body
+ response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
- manifest = manifest.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
- url = baseurl.lchop("")
- url = url.rchop(" ")
-
- if local
- uri = URI.parse(url)
- url = "#{uri.request_target}host/#{uri.host}/"
- end
+ if response.status_code != 200
+ haltf env, status_code: response.status_code
+ end
+ # Proxy URLs for video playback on invidious.
+ # Other API clients can get the original URLs by omiting `local=true`.
+ manifest = response.body.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
+ url = baseurl.lchop("").rchop(" ")
+ url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
"#{url} "
end
return manifest
end
- adaptive_fmts = video.adaptive_fmts
-
+ # Ditto, only proxify URLs if `local=true` is used
if local
- adaptive_fmts.each do |fmt|
- fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target)
+ video.adaptive_fmts.each do |fmt|
+ fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
end
end
- audio_streams = video.audio_streams
+ audio_streams = video.audio_streams.sort_by { |stream| {stream["bitrate"].as_i} }.reverse!
video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse!
manifest = XML.build(indent: " ", encoding: "UTF-8") do |xml|
@@ -56,17 +60,32 @@ module Invidious::Routes::API::Manifest
xml.element("Period") do
i = 0
- {"audio/mp4", "audio/webm"}.each do |mime_type|
+ {"audio/mp4"}.each do |mime_type|
mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
- xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do
- mime_streams.each do |fmt|
+ mime_streams.each do |fmt|
+ # OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
+ next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
+
+ audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any
+ lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und"
+ is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0
+ displayname = audio_track["displayName"]?.try &.as_s || "Unknown"
+ bitrate = fmt["bitrate"]
+
+ # Different representations of the same audio should be groupped into one AdaptationSet.
+ # However, most players don't support auto quality switching, so we have to trick them
+ # into providing a quality selector.
+ # See https://github.com/iv-org/invidious/issues/3074 for more details.
+ xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
+ xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate")
+
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
value: "2")
@@ -76,20 +95,22 @@ module Invidious::Routes::API::Manifest
end
end
end
+ i += 1
end
-
- i += 1
end
potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144}
- {"video/mp4", "video/webm"}.each do |mime_type|
+ {"video/mp4"}.each do |mime_type|
mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
heights = [] of Int32
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do
mime_streams.each do |fmt|
+ # OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
+ next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
+
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
@@ -160,8 +181,9 @@ module Invidious::Routes::API::Manifest
manifest = response.body
if local
- manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
- path = URI.parse(match).path
+ manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match|
+ uri = URI.parse(match)
+ path = uri.path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
@@ -190,7 +212,7 @@ module Invidious::Routes::API::Manifest
raw_params["fvip"] = fvip["fvip"]
end
- raw_params["local"] = "true"
+ raw_params["host"] = uri.host.not_nil!
"#{HOST_URL}/videoplayback?#{raw_params}"
end
diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr
index b559a01a..a35d2f2b 100644
--- a/src/invidious/routes/api/v1/authenticated.cr
+++ b/src/invidious/routes/api/v1/authenticated.cr
@@ -31,6 +31,88 @@ module Invidious::Routes::API::V1::Authenticated
env.response.status_code = 204
end
+ def self.export_invidious(env)
+ env.response.content_type = "application/json"
+ user = env.get("user").as(User)
+
+ return Invidious::User::Export.to_invidious(user)
+ end
+
+ def self.import_invidious(env)
+ user = env.get("user").as(User)
+
+ begin
+ if body = env.request.body
+ body = env.request.body.not_nil!.gets_to_end
+ else
+ body = "{}"
+ end
+ Invidious::User::Import.from_invidious(user, body)
+ rescue
+ end
+
+ env.response.status_code = 204
+ end
+
+ def self.get_history(env)
+ env.response.content_type = "application/json"
+ user = env.get("user").as(User)
+
+ page = env.params.query["page"]?.try &.to_i?.try &.clamp(0, Int32::MAX)
+ page ||= 1
+
+ max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
+ max_results ||= user.preferences.max_results
+ max_results ||= CONFIG.default_user_preferences.max_results
+
+ start_index = (page - 1) * max_results
+ if user.watched[start_index]?
+ watched = user.watched.reverse[start_index, max_results]
+ end
+ watched ||= [] of String
+
+ return watched.to_json
+ end
+
+ def self.mark_watched(env)
+ user = env.get("user").as(User)
+
+ if !user.preferences.watch_history
+ return error_json(409, "Watch history is disabled in preferences.")
+ end
+
+ id = env.params.url["id"]
+ if !id.match(/^[a-zA-Z0-9_-]{11}$/)
+ return error_json(400, "Invalid video id.")
+ end
+
+ Invidious::Database::Users.mark_watched(user, id)
+ env.response.status_code = 204
+ end
+
+ def self.mark_unwatched(env)
+ user = env.get("user").as(User)
+
+ if !user.preferences.watch_history
+ return error_json(409, "Watch history is disabled in preferences.")
+ end
+
+ id = env.params.url["id"]
+ if !id.match(/^[a-zA-Z0-9_-]{11}$/)
+ return error_json(400, "Invalid video id.")
+ end
+
+ Invidious::Database::Users.mark_unwatched(user, id)
+ env.response.status_code = 204
+ end
+
+ def self.clear_history(env)
+ user = env.get("user").as(User)
+
+ Invidious::Database::Users.clear_watch_history(user)
+ env.response.status_code = 204
+ end
+
def self.feed(env)
env.response.content_type = "application/json"
@@ -96,10 +178,6 @@ module Invidious::Routes::API::V1::Authenticated
Invidious::Database::Users.subscribe_channel(user, ucid)
end
- # For Google accounts, access tokens don't have enough information to
- # make a request on the user's behalf, which is why we don't sync with
- # YouTube.
-
env.response.status_code = 204
end
@@ -226,8 +304,8 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(403, "Invalid user")
end
- if playlist.index.size >= 500
- return error_json(400, "Playlist cannot have more than 500 videos")
+ if playlist.index.size >= CONFIG.playlist_length_limit
+ return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
video_id = env.params.json["videoId"].try &.as(String)
@@ -237,6 +315,8 @@ module Invidious::Routes::API::V1::Authenticated
begin
video = get_video(video_id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index c4d6643a..a940ee68 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -1,29 +1,44 @@
module Invidious::Routes::API::V1::Channels
- def self.home(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "newest"
-
+ # Macro to avoid duplicating some code below
+ # This sets the `channel` variable, or handles Exceptions.
+ private macro get_channel
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
+ end
- page = 1
- if channel.auto_generated
- videos = [] of SearchVideo
- count = 0
+ def self.home(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve "sort by" setting from URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+
+ if channel.is_age_gated
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
+ videos = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ videos = [] of PlaylistVideo
+ end
+ next_continuation = nil
else
begin
- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
rescue ex
return error_json(500, ex)
end
@@ -80,11 +95,15 @@ module Invidious::Routes::API::V1::Channels
json.field "joined", channel.joined.to_unix
json.field "autoGenerated", channel.auto_generated
+ json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html
json.field "allowedRegions", channel.allowed_regions
+ json.field "tabs", channel.tabs
+ json.field "tags", channel.tags
+ json.field "authorVerified", channel.verified
json.field "latestVideos" do
json.array do
@@ -98,31 +117,13 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
- related_channels = fetch_related_channels(channel)
+ related_channels, _ = fetch_related_channels(channel)
rescue ex
- related_channels = [] of AboutRelatedChannel
+ related_channels = [] of SearchChannel
end
related_channels.each do |related_channel|
- json.object do
- json.field "author", related_channel.author
- json.field "authorId", related_channel.ucid
- json.field "authorUrl", related_channel.author_url
-
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
- end
+ related_channel.to_json(locale, json)
end
end
end # relatedChannels
@@ -132,59 +133,147 @@ module Invidious::Routes::API::V1::Channels
end
def self.latest(env)
- locale = env.get("preferences").as(Preferences).locale
+ # Remove parameters that could affect this endpoint's behavior
+ env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
+ env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
-
- begin
- videos = get_latest_videos(ucid)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.array do
- videos.each do |video|
- video.to_json(locale, json)
- end
- end
- end
+ return self.videos(env)
end
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
- ucid = env.params.url["ucid"]
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
- sort_by = env.params.query["sort"]?.try &.downcase
- sort_by ||= env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "newest"
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
- rescue ex
- return error_json(500, ex)
+ # Retrieve some URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ continuation = env.params.query["continuation"]?
+
+ if channel.is_age_gated
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
+ videos = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ videos = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
end
- begin
- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.array do
- videos.each do |video|
- video.to_json(locale, json)
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.shorts(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve continuation from URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ continuation = env.params.query["continuation"]?
+
+ if channel.is_age_gated
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
+ videos = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ videos = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ begin
+ videos, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.streams(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve continuation from URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ continuation = env.params.query["continuation"]?
+
+ if channel.is_age_gated
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
+ videos = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ videos = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
end
end
end
@@ -200,16 +289,11 @@ module Invidious::Routes::API::V1::Channels
env.params.query["sort_by"]?.try &.downcase ||
"last"
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
- rescue ex
- return error_json(500, ex)
- end
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
JSON.build do |json|
json.object do
@@ -221,7 +305,94 @@ module Invidious::Routes::API::V1::Channels
end
end
- json.field "continuation", continuation
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.podcasts(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ continuation = env.params.query["continuation"]?
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ items, next_continuation = fetch_channel_podcasts(channel.ucid, channel.author, continuation)
+
+ JSON.build do |json|
+ json.object do
+ json.field "playlists" do
+ json.array do
+ items.each do |item|
+ item.to_json(locale, json) if item.is_a?(SearchPlaylist)
+ end
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.releases(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ continuation = env.params.query["continuation"]?
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ items, next_continuation = fetch_channel_releases(channel.ucid, channel.author, continuation)
+
+ JSON.build do |json|
+ json.object do
+ json.field "playlists" do
+ json.array do
+ items.each do |item|
+ item.to_json(locale, json) if item.is_a?(SearchPlaylist)
+ end
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.courses(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ continuation = env.params.query["continuation"]?
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation)
+
+ JSON.build do |json|
+ json.object do
+ json.field "playlists" do
+ json.array do
+ items.each do |item|
+ item.to_json(locale, json) if item.is_a?(SearchPlaylist)
+ end
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
end
end
end
@@ -249,20 +420,108 @@ module Invidious::Routes::API::V1::Channels
end
end
- def self.search(env)
+ def self.post(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+ id = env.params.url["id"].to_s
+ ucid = env.params.query["ucid"]?
+
+ thin_mode = env.params.query["thin_mode"]?
+ thin_mode = thin_mode == "true"
+
+ format = env.params.query["format"]?
+ format ||= "json"
+
+ if ucid.nil?
+ response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
+ return error_json(400, "Invalid post ID") if response["error"]?
+ ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+ else
+ ucid = ucid.to_s
+ end
+
+ begin
+ fetch_channel_community_post(ucid, id, locale, format, thin_mode)
+ rescue ex
+ return error_json(500, ex)
+ end
+ end
+
+ def self.post_comments(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
+ id = env.params.url["id"]
+
+ thin_mode = env.params.query["thin_mode"]?
+ thin_mode = thin_mode == "true"
+
+ format = env.params.query["format"]?
+ format ||= "json"
+
+ continuation = env.params.query["continuation"]?
+
+ case continuation
+ when nil, ""
+ ucid = env.params.query["ucid"]
+ comments = Comments.fetch_community_post_comments(ucid, id)
+ else
+ comments = YoutubeAPI.browse(continuation: continuation)
+ end
+ return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true)
+ end
+
+ def self.channels(env)
+ locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
- query = env.params.query["q"]?
- query ||= ""
+ env.response.content_type = "application/json"
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ continuation = env.params.query["continuation"]?
+
+ begin
+ items, next_continuation = fetch_related_channels(channel, continuation)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ JSON.build do |json|
+ json.object do
+ json.field "relatedChannels" do
+ json.array do
+ items.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.search(env)
+ locale = env.get("preferences").as(Preferences).locale
+ region = env.params.query["region"]?
+
+ env.response.content_type = "application/json"
+
+ query = Invidious::Search::Query.new(env.params.query, :channel, region)
+
+ # Required because we can't (yet) pass multiple parameter to the
+ # `Search::Query` initializer (in this case, an URL segment)
+ query.channel = env.params.url["ucid"]
+
+ begin
+ search_results = query.process
+ rescue ex
+ return error_json(400, ex)
+ end
- search_results = channel_search(query, page, ucid)
JSON.build do |json|
json.array do
search_results.each do |item|
diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr
index 41865f34..fea2993c 100644
--- a/src/invidious/routes/api/v1/feeds.cr
+++ b/src/invidious/routes/api/v1/feeds.cr
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds
if !CONFIG.popular_enabled
error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
- haltf env, 400, error_message
+ haltf env, 403, error_message
end
JSON.build do |json|
diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr
index 844fedb8..4f5b58da 100644
--- a/src/invidious/routes/api/v1/misc.cr
+++ b/src/invidious/routes/api/v1/misc.cr
@@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc
if !CONFIG.statistics_enabled
return {"software" => SOFTWARE}.to_json
else
+ # Calculate playback success rate
+ if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
+ tracker = tracker.as(Hash(String, Int64 | Float64))
+
+ if !tracker.empty?
+ total_requests = tracker["totalRequests"]
+ success_count = tracker["successfulRequests"]
+
+ if total_requests.zero?
+ tracker["ratio"] = 1_i64
+ else
+ tracker["ratio"] = (success_count / (total_requests)).round(2)
+ end
+ end
+ end
+
return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
end
@@ -26,6 +42,9 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
+ listen_param = env.params.query["listen"]?
+ listen = (listen_param == "true" || listen_param == "1")
+
if plid.starts_with? "RD"
return env.redirect "/api/v1/mixes/#{plid}"
end
@@ -58,7 +77,9 @@ module Invidious::Routes::API::V1::Misc
response = playlist.to_json(offset, video_id: video_id)
json_response = JSON.parse(response)
- if json_response["videos"].as_a[0]["index"] != offset
+ if json_response["videos"].as_a.empty?
+ json_response = JSON.parse(response)
+ elsif json_response["videos"].as_a[0]["index"] != offset
offset = json_response["videos"].as_a[0]["index"].as_i
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback)
@@ -67,7 +88,7 @@ module Invidious::Routes::API::V1::Misc
end
if format == "html"
- playlist_html = template_playlist(json_response)
+ playlist_html = template_playlist(json_response, listen)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = {
@@ -93,6 +114,9 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
+ listen_param = env.params.query["listen"]?
+ listen = (listen_param == "true" || listen_param == "1")
+
begin
mix = fetch_mix(rdid, continuation, locale: locale)
@@ -123,9 +147,7 @@ module Invidious::Routes::API::V1::Misc
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "videoThumbnails" do
- json.array do
- generate_thumbnails(json, video.id)
- end
+ Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
json.field "index", video.index
@@ -139,7 +161,7 @@ module Invidious::Routes::API::V1::Misc
if format == "html"
response = JSON.parse(response)
- playlist_html = template_mix(response)
+ playlist_html = template_mix(response, listen)
next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"]
response = {
@@ -150,4 +172,36 @@ module Invidious::Routes::API::V1::Misc
response
end
+
+ # resolve channel and clip urls, return the UCID
+ def self.resolve_url(env)
+ env.response.content_type = "application/json"
+ url = env.params.query["url"]?
+
+ return error_json(400, "Missing URL to resolve") if !url
+
+ begin
+ resolved_url = YoutubeAPI.resolve_url(url.as(String))
+ endpoint = resolved_url["endpoint"]
+ page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
+ if page_type == "WEB_PAGE_TYPE_UNKNOWN"
+ return error_json(400, "Unknown url")
+ end
+
+ sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
+ params = sub_endpoint.try &.dig?("params")
+ rescue ex
+ return error_json(500, ex)
+ end
+ JSON.build do |json|
+ json.object do
+ json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
+ json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
+ json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
+ json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
+ json.field "params", params.try &.as_s
+ json.field "pageType", page_type
+ end
+ end
+ end
end
diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 0b0853b1..59a30745 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search
env.response.content_type = "application/json"
- query = env.params.query["q"]?
- query ||= ""
-
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
-
- sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "relevance"
-
- date = env.params.query["date"]?.try &.downcase
- date ||= ""
-
- duration = env.params.query["duration"]?.try &.downcase
- duration ||= ""
-
- features = env.params.query["features"]?.try &.split(",").map(&.downcase)
- features ||= [] of String
-
- content_type = env.params.query["type"]?.try &.downcase
- content_type ||= "video"
+ query = Invidious::Search::Query.new(env.params.query, :regular, region)
begin
- search_params = produce_search_params(page, sort_by, date, content_type, duration, features)
+ search_results = query.process
rescue ex
return error_json(400, ex)
end
- search_results = search(query, search_params, region)
JSON.build do |json|
json.array do
search_results.each do |item|
@@ -43,20 +23,21 @@ module Invidious::Routes::API::V1::Search
end
def self.search_suggestions(env)
- locale = env.get("preferences").as(Preferences).locale
- region = env.params.query["region"]?
+ preferences = env.get("preferences").as(Preferences)
+ region = env.params.query["region"]? || preferences.region
env.response.content_type = "application/json"
- query = env.params.query["q"]?
- query ||= ""
+ query = env.params.query["q"]? || ""
begin
- headers = HTTP::Headers{":authority" => "suggestqueries.google.com"}
- response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body
+ client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
+ url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
- body = response[35..-2]
- body = JSON.parse(body).as_a
+ response = client.get(url).body
+ client.close
+
+ body = JSON.parse(response[19..-2]).as_a
suggestions = body[1].as_a[0..-2]
JSON.build do |json|
@@ -75,4 +56,32 @@ module Invidious::Routes::API::V1::Search
return error_json(500, ex)
end
end
+
+ def self.hashtag(env)
+ hashtag = env.params.url["hashtag"]
+
+ page = env.params.query["page"]?.try &.to_i? || 1
+
+ locale = env.get("preferences").as(Preferences).locale
+ region = env.params.query["region"]?
+ env.response.content_type = "application/json"
+
+ begin
+ results = Invidious::Hashtag.fetch(hashtag, page, region)
+ rescue ex
+ return error_json(400, ex)
+ end
+
+ JSON.build do |json|
+ json.object do
+ json.field "results" do
+ json.array do
+ results.each do |item|
+ item.to_json(locale, json)
+ end
+ end
+ end
+ end
+ end
+ end
end
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index a9f891f5..6a3eb8ae 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -1,3 +1,5 @@
+require "html"
+
module Invidious::Routes::API::V1::Videos
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
@@ -6,17 +8,19 @@ module Invidious::Routes::API::V1::Videos
id = env.params.url["id"]
region = env.params.query["region"]?
+ proxy = {"1", "true"}.any? &.== env.params.query["local"]?
begin
video = get_video(id, region: region)
- rescue ex : VideoRedirect
- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
- video.to_json(locale, nil)
+ return JSON.build do |json|
+ Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
+ end
end
def self.captions(env)
@@ -39,9 +43,8 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
- rescue ex : VideoRedirect
- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
+ rescue ex : NotFoundException
+ haltf env, 404
rescue ex
haltf env, 500
end
@@ -86,61 +89,77 @@ module Invidious::Routes::API::V1::Videos
caption = caption[0]
end
- url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
+ if CONFIG.use_innertube_for_captions
+ params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
- # Auto-generated captions often have cues that aren't aligned properly with the video,
- # as well as some other markup that makes it cumbersome, so we try to fix that here
- if caption.name.includes? "auto-generated"
- caption_xml = YT_POOL.client &.get(url).body
- caption_xml = XML.parse(caption_xml)
+ transcript = Invidious::Videos::Transcript.from_raw(
+ YoutubeAPI.get_transcript(params),
+ caption.language_code,
+ caption.auto_generated
+ )
- webvtt = String.build do |str|
- str << <<-END_VTT
- WEBVTT
- Kind: captions
- Language: #{tlang || caption.language_code}
+ webvtt = transcript.to_vtt
+ else
+ # Timedtext API handling
+ url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
+ # Auto-generated captions often have cues that aren't aligned properly with the video,
+ # as well as some other markup that makes it cumbersome, so we try to fix that here
+ if caption.name.includes? "auto-generated"
+ caption_xml = YT_POOL.client &.get(url).body
- END_VTT
+ settings_field = {
+ "Kind" => "captions",
+ "Language" => "#{tlang || caption.language_code}",
+ }
- caption_nodes = caption_xml.xpath_nodes("//transcript/text")
- caption_nodes.each_with_index do |node, i|
- start_time = node["start"].to_f.seconds
- duration = node["dur"]?.try &.to_f.seconds
- duration ||= start_time
+ if caption_xml.starts_with?(" i + 1
- end_time = caption_nodes[i + 1]["start"].to_f.seconds
- else
- end_time = start_time + duration
+ webvtt = WebVTT.build(settings_field) do |builder|
+ caption_nodes = caption_xml.xpath_nodes("//transcript/text")
+ caption_nodes.each_with_index do |node, i|
+ start_time = node["start"].to_f.seconds
+ duration = node["dur"]?.try &.to_f.seconds
+ duration ||= start_time
+
+ if caption_nodes.size > i + 1
+ end_time = caption_nodes[i + 1]["start"].to_f.seconds
+ else
+ end_time = start_time + duration
+ end
+
+ text = HTML.unescape(node.content)
+ text = text.gsub(//, "")
+ text = text.gsub(/<\/font>/, "")
+ if md = text.match(/(?.*) : (?.*)/)
+ text = "#{md["text"]} "
+ end
+
+ builder.cue(start_time, end_time, text)
+ end
end
+ end
+ else
+ uri = URI.parse(url)
+ query_params = uri.query_params
+ query_params["fmt"] = "vtt"
+ uri.query_params = query_params
+ webvtt = YT_POOL.client &.get(uri.request_target).body
- start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
- end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
-
- text = HTML.unescape(node.content)
- text = text.gsub(//, "")
- text = text.gsub(/<\/font>/, "")
- if md = text.match(/(?.*) : (?.*)/)
- text = "#{md["text"]} "
- end
-
- str << <<-END_CUE
- #{start_time} --> #{end_time}
- #{text}
-
-
- END_CUE
+ if webvtt.starts_with?(" [0-9:.]{12}).+/, "\\1")
end
end
- else
- # Some captions have "align:[start/end]" and "position:[num]%"
- # attributes. Those are causing issues with VideoJS, which is unable
- # to properly align the captions on the video, so we remove them.
- #
- # See: https://github.com/iv-org/invidious/issues/2391
- webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
- .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end
if title = env.params.query["title"]?
@@ -164,22 +183,20 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
- rescue ex : VideoRedirect
- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
+ rescue ex : NotFoundException
+ haltf env, 404
rescue ex
haltf env, 500
end
- storyboards = video.storyboards
- width = env.params.query["width"]?
- height = env.params.query["height"]?
+ width = env.params.query["width"]?.try &.to_i
+ height = env.params.query["height"]?.try &.to_i
if !width && !height
response = JSON.build do |json|
json.object do
json.field "storyboards" do
- generate_storyboards(json, id, storyboards)
+ Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards)
end
end
end
@@ -189,43 +206,48 @@ module Invidious::Routes::API::V1::Videos
env.response.content_type = "text/vtt"
- storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" }
+ # Select a storyboard matching the user's provided width/height
+ storyboard = video.storyboards.select { |x| x.width == width || x.height == height }
+ haltf env, 404 if storyboard.empty?
- if storyboard.empty?
- haltf env, 404
- else
- storyboard = storyboard[0]
- end
+ # Alias variable, to make the code below esaier to read
+ sb = storyboard[0]
- String.build do |str|
- str << <<-END_VTT
- WEBVTT
- END_VTT
+ # Some base URL segments that we'll use to craft the final URLs
+ work_url = sb.proxied_url.dup
+ template_path = sb.proxied_url.path
- start_time = 0.milliseconds
- end_time = storyboard[:interval].milliseconds
+ # Initialize cue timing variables
+ # NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap
+ # (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000)
+ time_delta = sb.interval.milliseconds
+ start_time = 0.milliseconds
+ end_time = time_delta
- storyboard[:storyboard_count].times do |i|
- url = storyboard[:url]
- authority = /(i\d?).ytimg.com/.match(url).not_nil![1]?
- url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "")
- url = "#{HOST_URL}/sb/#{authority}/#{url}"
+ # Build a VTT file for VideoJS-vtt plugin
+ vtt_file = WebVTT.build do |vtt|
+ sb.images_count.times do |i|
+ # Replace the variable component part of the path
+ work_url.path = template_path.sub("$M", i)
- storyboard[:storyboard_height].times do |j|
- storyboard[:storyboard_width].times do |k|
- str << <<-END_CUE
- #{start_time}.000 --> #{end_time}.000
- #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}
+ sb.rows.times do |j|
+ sb.columns.times do |k|
+ # The URL fragment represents the offset of the thumbnail inside the storyboard image
+ work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}"
+ vtt.cue(start_time, end_time, work_url.to_s)
- END_CUE
-
- start_time += storyboard[:interval].milliseconds
- end_time += storyboard[:interval].milliseconds
+ start_time += time_delta
+ end_time += time_delta
end
end
end
end
+
+ # videojs-vtt-thumbnails is not compliant to the VTT specification, it
+ # doesn't unescape the HTML entities, so we have to do it here:
+ # TODO: remove this when we migrate to VideoJS 8
+ return HTML.unescape(vtt_file)
end
def self.annotations(env)
@@ -246,7 +268,7 @@ module Invidious::Routes::API::V1::Videos
if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
annotations = cached_annotation.annotations
else
- index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0')
+ index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
# IA doesn't handle leading hyphens,
# so we use https://archive.org/details/youtubeannotations_64
@@ -323,7 +345,9 @@ module Invidious::Routes::API::V1::Videos
sort_by ||= "top"
begin
- comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
+ comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
@@ -333,7 +357,7 @@ module Invidious::Routes::API::V1::Videos
sort_by ||= "confidence"
begin
- comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by)
+ comments, reddit_thread = Comments.fetch_reddit(id, sort_by: sort_by)
rescue ex
comments = nil
reddit_thread = nil
@@ -349,9 +373,9 @@ module Invidious::Routes::API::V1::Videos
return reddit_thread.to_json
else
- content_html = template_reddit_comments(comments, locale)
- content_html = fill_links(content_html, "https", "www.reddit.com")
- content_html = replace_links(content_html)
+ content_html = Frontend::Comments.template_reddit(comments, locale)
+ content_html = Comments.fill_links(content_html, "https", "www.reddit.com")
+ content_html = Comments.replace_links(content_html)
response = {
"title" => reddit_thread.title,
"permalink" => reddit_thread.permalink,
@@ -362,4 +386,133 @@ module Invidious::Routes::API::V1::Videos
end
end
end
+
+ def self.clips(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ clip_id = env.params.url["id"]
+ region = env.params.query["region"]?
+ proxy = {"1", "true"}.any? &.== env.params.query["local"]?
+
+ response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
+ return error_json(400, "Invalid clip ID") if response["error"]?
+
+ video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
+ return error_json(400, "Invalid clip ID") if video_id.nil?
+
+ start_time = nil
+ end_time = nil
+ clip_title = nil
+
+ if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
+ start_time, end_time, clip_title = parse_clip_parameters(params)
+ end
+
+ begin
+ video = get_video(video_id, region: region)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "startTime", start_time
+ json.field "endTime", end_time
+ json.field "clipTitle", clip_title
+ json.field "video" do
+ Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
+ end
+ end
+ end
+ end
+
+ # Fetches transcripts from YouTube
+ #
+ # Use the `lang` and `autogen` query parameter to select which transcript to fetch
+ # Request without any URL parameters to see all the available transcripts.
+ def self.transcripts(env)
+ env.response.content_type = "application/json"
+
+ id = env.params.url["id"]
+ lang = env.params.query["lang"]?
+ label = env.params.query["label"]?
+ auto_generated = env.params.query["autogen"]? ? true : false
+
+ # Return all available transcript options when none is given
+ if !label && !lang
+ begin
+ video = get_video(id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ response = JSON.build do |json|
+ # The amount of transcripts available to fetch is the
+ # same as the amount of captions available.
+ available_transcripts = video.captions
+
+ json.object do
+ json.field "transcripts" do
+ json.array do
+ available_transcripts.each do |transcript|
+ json.object do
+ json.field "label", transcript.name
+ json.field "languageCode", transcript.language_code
+ json.field "autoGenerated", transcript.auto_generated
+
+ if transcript.auto_generated
+ json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}&autogen"
+ else
+ json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}"
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ return response
+ end
+
+ # If lang is not given then we attempt to fetch
+ # the transcript through the given label
+ if lang.nil?
+ begin
+ video = get_video(id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ target_transcript = video.captions.select(&.name.== label)
+ if target_transcript.empty?
+ return error_json(404, NotFoundException.new("Requested transcript does not exist"))
+ else
+ target_transcript = target_transcript[0]
+ lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated
+ end
+ end
+
+ params = Invidious::Videos::Transcript.generate_param(id, lang, auto_generated)
+
+ begin
+ transcript = Invidious::Videos::Transcript.from_raw(
+ YoutubeAPI.get_transcript(params), lang, auto_generated
+ )
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return transcript.to_json
+ end
end
diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr
new file mode 100644
index 00000000..b5269668
--- /dev/null
+++ b/src/invidious/routes/before_all.cr
@@ -0,0 +1,133 @@
+module Invidious::Routes::BeforeAll
+ def self.handle(env)
+ preferences = Preferences.from_json("{}")
+
+ begin
+ if prefs_cookie = env.request.cookies["PREFS"]?
+ preferences = Preferences.from_json(URI.decode_www_form(prefs_cookie.value))
+ else
+ if language_header = env.request.headers["Accept-Language"]?
+ if language = ANG.language_negotiator.best(language_header, LOCALES.keys)
+ preferences.locale = language.header
+ end
+ end
+ end
+ rescue
+ preferences = Preferences.from_json("{}")
+ end
+
+ env.set "preferences", preferences
+ env.response.headers["X-XSS-Protection"] = "1; mode=block"
+ env.response.headers["X-Content-Type-Options"] = "nosniff"
+
+ # Only allow the pages at /embed/* to be embedded
+ if env.request.resource.starts_with?("/embed")
+ frame_ancestors = "'self' file: http: https:"
+ else
+ frame_ancestors = "'none'"
+ end
+
+ # TODO: Remove style-src's 'unsafe-inline', requires to remove all
+ # inline styles (, style=" [..] ")
+ env.response.headers["Content-Security-Policy"] = {
+ "default-src 'none'",
+ "script-src 'self'",
+ "style-src 'self' 'unsafe-inline'",
+ "img-src 'self' data:",
+ "font-src 'self' data:",
+ "connect-src 'self'",
+ "manifest-src 'self'",
+ "media-src 'self' blob:",
+ "child-src 'self' blob:",
+ "frame-src 'self'",
+ "frame-ancestors " + frame_ancestors,
+ }.join("; ")
+
+ env.response.headers["Referrer-Policy"] = "same-origin"
+
+ # Ask the chrom*-based browsers to disable FLoC
+ # See: https://blog.runcloud.io/google-floc/
+ env.response.headers["Permissions-Policy"] = "interest-cohort=()"
+
+ if (Kemal.config.ssl || CONFIG.https_only) && CONFIG.hsts
+ env.response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload"
+ end
+
+ return if {
+ "/sb/",
+ "/vi/",
+ "/s_p/",
+ "/yts/",
+ "/ggpht/",
+ "/api/manifest/",
+ "/videoplayback",
+ "/latest_version",
+ "/download",
+ }.any? { |r| env.request.resource.starts_with? r }
+
+ if env.request.cookies.has_key? "SID"
+ sid = env.request.cookies["SID"].value
+
+ if sid.starts_with? "v1:"
+ raise "Cannot use token as SID"
+ end
+
+ if email = Database::SessionIDs.select_email(sid)
+ user = Database::Users.select!(email: email)
+ csrf_token = generate_response(sid, {
+ ":authorize_token",
+ ":playlist_ajax",
+ ":signout",
+ ":subscription_ajax",
+ ":token_ajax",
+ ":watch_ajax",
+ }, HMAC_KEY, 1.week)
+
+ preferences = user.preferences
+ env.set "preferences", preferences
+
+ env.set "sid", sid
+ env.set "csrf_token", csrf_token
+ env.set "user", user
+ end
+ end
+
+ dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s
+ thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode.to_s
+ thin_mode = thin_mode == "true"
+ locale = env.params.query["hl"]? || preferences.locale
+
+ preferences.dark_mode = dark_mode
+ preferences.thin_mode = thin_mode
+ preferences.locale = locale
+ env.set "preferences", preferences
+
+ # Allow media resources to be loaded from google servers
+ # TODO: check if *.youtube.com can be removed
+ #
+ # `!preferences.local` has to be checked after setting and
+ # reading `preferences` from the "PREFS" cookie and
+ # saved user preferences from the database, otherwise
+ # `https://*.googlevideo.com:443 https://*.youtube.com:443`
+ # will not be set in the CSP header if
+ # `default_user_preferences.local` is set to true on the
+ # configuration file, causing preference “Proxy Videos”
+ # not to work while having it disabled and using medium quality.
+ if CONFIG.disabled?("local") || !preferences.local
+ env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"].gsub("media-src", "media-src https://*.googlevideo.com:443 https://*.youtube.com:443")
+ end
+
+ current_page = env.request.path
+ if env.request.query
+ query = HTTP::Params.parse(env.request.query.not_nil!)
+
+ if query["referer"]?
+ query["referer"] = get_referer(env, "/")
+ end
+
+ current_page += "?#{query}"
+ end
+
+ env.set "current_page", URI.encode_www_form(current_page)
+ end
+end
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index cd2e3323..508aa3e4 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -1,27 +1,32 @@
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Channels
+ # Redirection for unsupported routes ("tabs")
+ def self.redirect_home(env)
+ ucid = env.params.url["ucid"]
+ return env.redirect "/channel/#{URI.encode_www_form(ucid)}"
+ end
+
def self.home(env)
self.videos(env)
end
def self.videos(env)
data = self.fetch_basic_information(env)
- if !data.is_a?(Tuple)
- return data
- end
- locale, user, subscriptions, continuation, ucid, channel = data
+ return data if !data.is_a?(Tuple)
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
+ locale, user, subscriptions, continuation, ucid, channel = data
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
- sort_options = {"last", "oldest", "newest"}
sort_by ||= "last"
+ sort_options = {"last", "oldest", "newest"}
+
+ items, next_continuation = fetch_channel_playlists(
+ channel.ucid, channel.author, continuation, sort_by
+ )
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@@ -29,62 +34,224 @@ module Invidious::Routes::Channels
item.author
end
end
- items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
+ items = items.select(SearchPlaylist)
items.each(&.author = "")
else
- sort_options = {"newest", "oldest", "popular"}
- sort_by ||= "newest"
+ # Fetch items and continuation token
+ if channel.is_age_gated
+ sort_by = ""
+ sort_options = [] of String
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
+ items = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ items = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ sort_by ||= "newest"
+ sort_options = {"newest", "oldest", "popular"}
- count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ items, next_continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ end
end
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
+ templated "channel"
+ end
+
+ def self.shorts(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if !channel.tabs.includes? "shorts"
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ if channel.is_age_gated
+ sort_by = ""
+ sort_options = [] of String
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
+ items = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ items = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ sort_options = {"newest", "oldest", "popular"}
+
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ end
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
+ templated "channel"
+ end
+
+ def self.streams(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if !channel.tabs.includes? "streams"
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ if channel.is_age_gated
+ sort_by = ""
+ sort_options = [] of String
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
+ items = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ items = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ sort_options = {"newest", "oldest", "popular"}
+
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ end
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
def self.playlists(env)
data = self.fetch_basic_information(env)
- if !data.is_a?(Tuple)
- return data
- end
+ return data if !data.is_a?(Tuple)
+
locale, user, subscriptions, continuation, ucid, channel = data
sort_options = {"last", "oldest", "newest"}
sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "last"
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
- items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
+ items, next_continuation = fetch_channel_playlists(
+ channel.ucid, channel.author, continuation, (sort_by || "last")
+ )
+
+ items = items.select(SearchPlaylist)
items.each(&.author = "")
- templated "playlists"
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
+ templated "channel"
+ end
+
+ def self.podcasts(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ sort_by = ""
+ sort_options = [] of String
+
+ items, next_continuation = fetch_channel_podcasts(
+ channel.ucid, channel.author, continuation
+ )
+
+ items = items.select(SearchPlaylist)
+ items.each(&.author = "")
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Podcasts
+ templated "channel"
+ end
+
+ def self.releases(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ sort_by = ""
+ sort_options = [] of String
+
+ items, next_continuation = fetch_channel_releases(
+ channel.ucid, channel.author, continuation
+ )
+
+ items = items.select(SearchPlaylist)
+ items.each(&.author = "")
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Releases
+ templated "channel"
+ end
+
+ def self.courses(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ sort_by = ""
+ sort_options = [] of String
+
+ items, next_continuation = fetch_channel_courses(
+ channel.ucid, channel.author, continuation
+ )
+
+ items = items.select(SearchPlaylist)
+ items.each(&.author = "")
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Courses
+ templated "channel"
end
def self.community(env)
+ return env.redirect env.request.path.sub("posts", "community") if env.request.path.split("/").last == "posts"
+
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
locale, user, subscriptions, continuation, ucid, channel = data
+ # redirect to post page
+ if lb = env.params.query["lb"]?
+ env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}"
+ end
+
thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
- # sort_by = env.params.query["sort_by"]?.try &.downcase
- if !channel.tabs.includes? "community"
+ if !channel.tabs.includes? "community" && "posts"
return env.redirect "/channel/#{channel.ucid}"
end
+ # TODO: support sort options for community posts
+ sort_by = ""
+ sort_options = [] of String
+
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
env.response.status_code = 500
error_message = ex.message
+ rescue ex : NotFoundException
+ env.response.status_code = 404
+ error_message = ex.message
rescue ex
return error_template(500, ex)
end
@@ -92,6 +259,64 @@ module Invidious::Routes::Channels
templated "community"
end
+ def self.post(env)
+ # /post/{postId}
+ id = env.params.url["id"]
+ ucid = env.params.query["ucid"]?
+
+ prefs = env.get("preferences").as(Preferences)
+
+ locale = prefs.locale
+
+ thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode
+ thin_mode = thin_mode == "true"
+
+ nojs = env.params.query["nojs"]?
+
+ nojs ||= "0"
+ nojs = nojs == "1"
+
+ if !ucid.nil?
+ ucid = ucid.to_s
+ post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
+ else
+ # resolve the url to get the author's UCID
+ response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
+ return error_template(400, "Invalid post ID") if response["error"]?
+
+ ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+ post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
+ end
+
+ post_response = JSON.parse(post_response)
+
+ if nojs
+ comments = Comments.fetch_community_post_comments(ucid, id)
+ comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"]
+ end
+ templated "post"
+ end
+
+ def self.channels(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if channel.auto_generated
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ items, next_continuation = fetch_related_channels(channel, continuation)
+
+ # Featured/related channels can't be sorted
+ sort_options = [] of String
+ sort_by = nil
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
+ templated "channel"
+ end
+
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
@@ -102,6 +327,12 @@ module Invidious::Routes::Channels
env.redirect "/channel/#{ucid}"
end
+ private KNOWN_TABS = {
+ "home", "videos", "shorts", "streams", "podcasts",
+ "releases", "courses", "playlists", "community", "channels", "about",
+ "posts",
+ }
+
# Redirects brand url channels to a normal /channel/:ucid route
def self.brand_redirect(env)
locale = env.get("preferences").as(Preferences).locale
@@ -112,23 +343,29 @@ module Invidious::Routes::Channels
yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"]))
# Retrieves URL params that only Invidious uses
- invidious_url_params = URI::Params.encode(env.params.query.to_h.select!(["a", "u", "user"]))
+ invidious_url_params = env.params.query.dup
+ invidious_url_params.delete_all("a")
+ invidious_url_params.delete_all("u")
+ invidious_url_params.delete_all("user")
begin
resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}")
ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"]
rescue ex : InfoException | KeyError
- raise InfoException.new(translate(locale, "This channel does not exist."))
+ return error_template(404, translate(locale, "This channel does not exist."))
end
- selected_tab = env.request.path.split("/")[-1]
- if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
+ selected_tab = env.params.url["tab"]?
+
+ if KNOWN_TABS.includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"
end
- env.redirect url
+ url += "?#{invidious_url_params}" if !invidious_url_params.empty?
+
+ return env.redirect url
end
# Handles redirects for the /profile endpoint
@@ -141,7 +378,7 @@ module Invidious::Routes::Channels
user = env.params.query["user"]?
if !user
- raise InfoException.new("This channel does not exist.")
+ return error_template(404, "This channel does not exist.")
else
env.redirect "/user/#{user}#{uri_params}"
end
@@ -197,10 +434,13 @@ module Invidious::Routes::Channels
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
+ env.set "search", "channel:#{ucid} "
return {locale, user, subscriptions, continuation, ucid, channel}
end
end
diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr
index 207970b0..930e4915 100644
--- a/src/invidious/routes/embed.cr
+++ b/src/invidious/routes/embed.cr
@@ -2,16 +2,25 @@
module Invidious::Routes::Embed
def self.redirect(env)
+ locale = env.get("preferences").as(Preferences).locale
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
+ if videos.empty?
+ url = "/playlist?list=#{plid}"
+ raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
+ end
+
+ first_playlist_video = videos[0].as(PlaylistVideo)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{videos[0].id}?#{env.params.query}"
+ url = "/embed/#{first_playlist_video}?#{env.params.query}"
if env.params.query.size > 0
url += "?#{env.params.query}"
@@ -24,6 +33,7 @@ module Invidious::Routes::Embed
end
def self.show(env)
+ locale = env.get("preferences").as(Preferences).locale
id = env.params.url["id"]
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
@@ -60,11 +70,19 @@ module Invidious::Routes::Embed
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
+ if videos.empty?
+ url = "/playlist?list=#{plid}"
+ raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
+ end
+
+ first_playlist_video = videos[0].as(PlaylistVideo)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{videos[0].id}"
+ url = "/embed/#{first_playlist_video.id}"
elsif video_series
url = "/embed/#{video_series.shift}"
env.params.query["playlist"] = video_series.join(",")
@@ -117,8 +135,8 @@ module Invidious::Routes::Embed
begin
video = get_video(id, region: params.region)
- rescue ex : VideoRedirect
- return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
@@ -133,7 +151,7 @@ module Invidious::Routes::Embed
# PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
# end
- if notifications && notifications.includes? id
+ if CONFIG.enable_user_notifications && notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
@@ -143,10 +161,12 @@ module Invidious::Routes::Embed
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
end
+ # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -187,6 +207,14 @@ module Invidious::Routes::Embed
return env.redirect url
end
+ if CONFIG.invidious_companion.present?
+ invidious_companion = CONFIG.invidious_companion.sample
+ env.response.headers["Content-Security-Policy"] =
+ env.response.headers["Content-Security-Policy"]
+ .gsub("media-src", "media-src #{invidious_companion.public_url}")
+ .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
+ end
+
rendered "embed"
end
end
diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr
new file mode 100644
index 00000000..1e9ab44e
--- /dev/null
+++ b/src/invidious/routes/errors.cr
@@ -0,0 +1,52 @@
+module Invidious::Routes::ErrorRoutes
+ def self.error_404(env)
+ # Workaround for #3117
+ if HOST_URL.empty? && env.request.path.starts_with?("/v1/storyboards/sb")
+ return env.redirect "#{env.request.path[15..]}?#{env.params.query}"
+ end
+
+ if md = env.request.path.match(/^\/(?([a-zA-Z0-9_-]{11})|(\w+))$/)
+ item = md["id"]
+
+ # Check if item is branding URL e.g. https://youtube.com/gaming
+ response = YT_POOL.client &.get("/#{item}")
+
+ if response.status_code == 301
+ response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target)
+ end
+
+ if response.body.empty?
+ env.response.headers["Location"] = "/"
+ haltf env, status_code: 302
+ end
+
+ html = XML.parse_html(response.body)
+ ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
+
+ if ucid
+ env.response.headers["Location"] = "/channel/#{ucid}"
+ haltf env, status_code: 302
+ end
+
+ params = [] of String
+ env.params.query.each do |k, v|
+ params << "#{k}=#{v}"
+ end
+ params = params.join("&")
+
+ url = "/watch?v=#{item}"
+ if !params.empty?
+ url += "&#{params}"
+ end
+
+ # Check if item is video ID
+ if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
+ env.response.headers["Location"] = url
+ haltf env, status_code: 302
+ end
+ end
+
+ env.response.headers["Location"] = "/"
+ haltf env, status_code: 302
+ end
+end
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index f7f7b426..070c96eb 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -83,10 +83,6 @@ module Invidious::Routes::Feeds
headers = HTTP::Headers.new
headers["Cookie"] = env.request.headers["Cookie"]
- if !user.password
- user, sid = get_user(sid, headers)
- end
-
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
@@ -96,14 +92,20 @@ module Invidious::Routes::Feeds
videos, notifications = get_subscription_feed(user, max_results, page)
- # "updated" here is used for delivering new notifications, so if
- # we know a user has looked at their feed e.g. in the past 10 minutes,
- # they've already seen a video posted 20 minutes ago, and don't need
- # to be notified.
- Invidious::Database::Users.clear_notifications(user)
- user.notifications = [] of String
+ if CONFIG.enable_user_notifications
+ # "updated" here is used for delivering new notifications, so if
+ # we know a user has looked at their feed e.g. in the past 10 minutes,
+ # they've already seen a video posted 20 minutes ago, and don't need
+ # to be notified.
+ Invidious::Database::Users.clear_notifications(user)
+ user.notifications = [] of String
+ end
env.set "user", user
+ # Used for pagination links
+ base_url = "/feed/subscriptions"
+ base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
+
templated "feeds/subscriptions"
end
@@ -131,78 +133,91 @@ module Invidious::Routes::Feeds
end
watched ||= [] of String
+ # Used for pagination links
+ base_url = "/feed/history"
+ base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
+
templated "feeds/history"
end
# RSS feeds
def self.rss_channel(env)
- locale = env.get("preferences").as(Preferences).locale
-
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
- ucid = env.params.url["ucid"]
+ if env.params.url["ucid"].matches?(/^[\w-]+$/)
+ ucid = env.params.url["ucid"]
+ else
+ return error_atom(400, InfoException.new("Invalid channel ucid provided."))
+ end
params = HTTP::Params.parse(env.params.query["params"]? || "")
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
- rescue ex
- return error_atom(500, ex)
- end
+ namespaces = {
+ "yt" => "http://www.youtube.com/xml/schemas/2015",
+ "media" => "http://search.yahoo.com/mrss/",
+ "default" => "http://www.w3.org/2005/Atom",
+ }
- response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
- rss = XML.parse_html(response.body)
+ response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
+ return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
+ rss = XML.parse(response.body)
- videos = rss.xpath_nodes("//feed/entry").map do |entry|
- video_id = entry.xpath_node("videoid").not_nil!.content
- title = entry.xpath_node("title").not_nil!.content
+ videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
+ video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+ title = entry.xpath_node("default:title", namespaces).not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
+ published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
- author = entry.xpath_node("author/name").not_nil!.content
- ucid = entry.xpath_node("channelid").not_nil!.content
- description_html = entry.xpath_node("group/description").not_nil!.to_s
- views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
+ author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
+ video_ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+ description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s
+ views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64
SearchVideo.new({
title: title,
id: video_id,
author: author,
- ucid: ucid,
+ ucid: video_ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
- live_now: false,
- paid: false,
- premium: false,
premiere_timestamp: nil,
+ author_verified: false,
+ author_thumbnail: nil,
+ badges: VideoBadges::None,
})
end
+ author = ""
+ author = videos[0].author if videos.size > 0
+
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
- xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
- xml.element("yt:channelId") { xml.text channel.ucid }
- xml.element("icon") { xml.text channel.author_thumbnail }
- xml.element("title") { xml.text channel.author }
- xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
+ xml.element("id") { xml.text "yt:channel:#{ucid}" }
+ xml.element("yt:channelId") { xml.text ucid }
+ xml.element("title") { xml.text author }
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{ucid}")
xml.element("author") do
- xml.element("name") { xml.text channel.author }
- xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
+ xml.element("name") { xml.text author }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
+ end
+
+ xml.element("image") do
+ xml.element("url") { xml.text "" }
+ xml.element("title") { xml.text author }
+ xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
end
videos.each do |video|
- video.to_xml(channel.auto_generated, params, xml)
+ video.to_xml(false, params, xml)
end
end
end
@@ -281,7 +296,13 @@ module Invidious::Routes::Feeds
xml.element("name") { xml.text playlist.author }
end
- videos.each &.to_xml(xml)
+ videos.each do |video|
+ if video.is_a? PlaylistVideo
+ video.to_xml(xml)
+ else
+ video.to_xml(env, locale, xml)
+ end
+ end
end
end
else
@@ -290,8 +311,9 @@ module Invidious::Routes::Feeds
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
- document = XML.parse(response.body)
+ return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
+ document = XML.parse(response.body)
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
node.attributes.each do |attribute|
case attribute.name
@@ -386,22 +408,23 @@ module Invidious::Routes::Feeds
end
spawn do
- rss = XML.parse_html(body)
- rss.xpath_nodes("//feed/entry").each do |entry|
- id = entry.xpath_node("videoid").not_nil!.content
- author = entry.xpath_node("author/name").not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
+ # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow?
+ namespaces = {
+ "yt" => "http://www.youtube.com/xml/schemas/2015",
+ "default" => "http://www.w3.org/2005/Atom",
+ }
+ rss = XML.parse(body)
+ rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
+ id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+ author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
+ published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
- video = get_video(id, force_refresh: true)
-
- # Deliver notifications to `/api/v1/auth/notifications`
- payload = {
- "topic" => video.ucid,
- "videoId" => video.id,
- "published" => published.to_unix,
- }.to_json
- PG_DB.exec("NOTIFY notifications, E'#{payload}'")
+ begin
+ video = get_video(id, force_refresh: true)
+ rescue
+ next # skip this video since it raised an exception (e.g. it is a scheduled live event)
+ end
video = ChannelVideo.new({
id: id,
@@ -417,7 +440,9 @@ module Invidious::Routes::Feeds
})
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
- Invidious::Database::Users.add_notification(video) if was_insert
+ if was_insert
+ NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
+ end
end
end
diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 594a7869..51d85dfe 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -3,17 +3,7 @@ module Invidious::Routes::Images
def self.ggpht(env)
url = env.request.path.lchop("/ggpht")
- headers = (
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- HTTP::Headers{":authority" => "yt3.ggpht.com"}
- else
- HTTP::Headers.new
- end
- {% else %}
- HTTP::Headers.new
- {% end %}
- )
+ headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
@@ -21,43 +11,10 @@ module Invidious::Routes::Images
end
end
- # We're encapsulating this into a proc in order to easily reuse this
- # portion of the code for each request block below.
- request_proc = ->(response : HTTP::Client::Response) {
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300
- env.response.headers.delete("Transfer-Encoding")
- return
- end
-
- proxy_file(response, env)
- }
-
begin
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- YT_POOL.client &.get(url, headers) do |resp|
- return request_proc.call(resp)
- end
- else
- HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- {% end %}
+ GGPHT_POOL.client &.get(url, headers) do |resp|
+ return self.proxy_image(env, resp)
+ end
rescue ex
end
end
@@ -78,51 +35,17 @@ module Invidious::Routes::Images
headers = HTTP::Headers.new
- {% unless flag?(:disable_quic) %}
- headers[":authority"] = "#{authority}.ytimg.com"
- {% end %}
-
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
- request_proc = ->(response : HTTP::Client::Response) {
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Connection"] = "close"
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- proxy_file(response, env)
- }
-
begin
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- YT_POOL.client &.get(url, headers) do |resp|
- return request_proc.call(resp)
- end
- else
- HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- {% end %}
+ get_ytimg_pool(authority).client &.get(url, headers) do |resp|
+ env.response.headers["Connection"] = "close"
+ return self.proxy_image(env, resp)
+ end
rescue ex
end
end
@@ -133,17 +56,7 @@ module Invidious::Routes::Images
name = env.params.url["name"]
url = env.request.resource
- headers = (
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- HTTP::Headers{":authority" => "i9.ytimg.com"}
- else
- HTTP::Headers.new
- end
- {% else %}
- HTTP::Headers.new
- {% end %}
- )
+ headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
@@ -151,40 +64,10 @@ module Invidious::Routes::Images
end
end
- request_proc = ->(response : HTTP::Client::Response) {
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300 && response.status_code != 404
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- proxy_file(response, env)
- }
-
begin
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- YT_POOL.client &.get(url, headers) do |resp|
- return request_proc.call(resp)
- end
- else
- HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- {% end %}
+ get_ytimg_pool("i9").client &.get(url, headers) do |resp|
+ return self.proxy_image(env, resp)
+ end
rescue ex
end
end
@@ -223,41 +106,15 @@ module Invidious::Routes::Images
id = env.params.url["id"]
name = env.params.url["name"]
- headers = (
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- HTTP::Headers{":authority" => "i.ytimg.com"}
- else
- HTTP::Headers.new
- end
- {% else %}
- HTTP::Headers.new
- {% end %}
- )
+ headers = HTTP::Headers.new
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
- # Logic here is short enough that manually typing them out should be fine.
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
- name = thumb[:url] + ".jpg"
- break
- end
- else
- if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
- name = thumb[:url] + ".jpg"
- break
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
- name = thumb[:url] + ".jpg"
- break
- end
- {% end %}
+ if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
+ name = thumb[:url] + ".jpg"
+ break
+ end
end
end
@@ -269,41 +126,28 @@ module Invidious::Routes::Images
end
end
- request_proc = ->(response : HTTP::Client::Response) {
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300 && response.status_code != 404
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- proxy_file(response, env)
- }
-
begin
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- YT_POOL.client &.get(url, headers) do |resp|
- return request_proc.call(resp)
- end
- else
- HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- {% end %}
+ get_ytimg_pool("i").client &.get(url, headers) do |resp|
+ return self.proxy_image(env, resp)
+ end
rescue ex
end
end
+
+ private def self.proxy_image(env, response)
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ return proxy_file(response, env)
+ end
end
diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr
index 99fc13a2..e7de5018 100644
--- a/src/invidious/routes/login.cr
+++ b/src/invidious/routes/login.cr
@@ -6,14 +6,14 @@ module Invidious::Routes::Login
user = env.get? "user"
- return env.redirect "/feed/subscriptions" if user
+ referer = get_referer(env, "/feed/subscriptions")
+
+ return env.redirect referer if user
if !CONFIG.login_enabled
return error_template(400, "Login has been disabled by administrator.")
end
- referer = get_referer(env, "/feed/subscriptions")
-
email = nil
password = nil
captcha = nil
@@ -21,12 +21,6 @@ module Invidious::Routes::Login
account_type = env.params.query["type"]?
account_type ||= "invidious"
- captcha_type = env.params.query["captcha"]?
- captcha_type ||= "image"
-
- tfa = env.params.query["tfa"]?
- prompt = nil
-
templated "user/login"
end
@@ -47,283 +41,18 @@ module Invidious::Routes::Login
account_type ||= "invidious"
case account_type
- when "google"
- tfa_code = env.params.body["tfa"]?.try &.lchop("G-")
- traceback = IO::Memory.new
-
- # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
- begin
- client = nil # Declare variable
- {% unless flag?(:disable_quic) %}
- client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL)
- {% else %}
- client = HTTP::Client.new(LOGIN_URL)
- {% end %}
-
- headers = HTTP::Headers.new
-
- login_page = client.get("/ServiceLogin")
- headers = login_page.cookies.add_request_headers(headers)
-
- lookup_req = {
- email, nil, [] of String, nil, "US", nil, nil, 2, false, true,
- {nil, nil,
- {2, 1, nil, 1,
- "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
- nil, [] of String, 4},
- 1,
- {nil, nil, [] of String},
- nil, nil, nil, true,
- },
- email,
- }.to_json
-
- traceback << "Getting lookup..."
-
- headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
- headers["Google-Accounts-XSRF"] = "1"
-
- response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
- lookup_results = JSON.parse(response.body[5..-1])
-
- traceback << "done, returned #{response.status_code}.
"
-
- user_hash = lookup_results[0][2]
-
- if token = env.params.body["token"]?
- answer = env.params.body["answer"]?
- captcha = {token, answer}
- else
- captcha = nil
- end
-
- challenge_req = {
- user_hash, nil, 1, nil,
- {1, nil, nil, nil,
- {password, captcha, true},
- },
- {nil, nil,
- {2, 1, nil, 1,
- "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
- nil, [] of String, 4},
- 1,
- {nil, nil, [] of String},
- nil, nil, nil, true,
- },
- }.to_json
-
- traceback << "Getting challenge..."
-
- response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req))
- headers = response.cookies.add_request_headers(headers)
- challenge_results = JSON.parse(response.body[5..-1])
-
- traceback << "done, returned #{response.status_code}.
"
-
- headers["Cookie"] = URI.decode_www_form(headers["Cookie"])
-
- if challenge_results[0][3]?.try &.== 7
- return error_template(423, "Account has temporarily been disabled")
- end
-
- if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s
- account_type = "google"
- captcha_type = "image"
- prompt = nil
- tfa = tfa_code
- captcha = {tokens: [token], question: ""}
-
- return templated "user/login"
- end
-
- if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED"
- return error_template(401, "Incorrect password")
- end
-
- prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]?
- if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type
- traceback << "Handling prompt #{prompt_type}.
"
- case prompt_type
- when "TWO_STEP_VERIFICATION"
- prompt_type = 2
- else # "LOGIN_CHALLENGE"
- prompt_type = 4
- end
-
- # Prefer Authenticator app and SMS over unsupported protocols
- if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2
- tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0]
-
- traceback << "Selecting challenge #{tfa[8]}..."
- select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json
-
- tl = challenge_results[1][2]
-
- tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body
- tfa = tfa[5..-1]
- tfa = JSON.parse(tfa)[0][-1]
-
- traceback << "done.
"
- else
- traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.
"
- tfa = challenge_results[0][-1][0][0]
- end
-
- if tfa[5] == "QUOTA_EXCEEDED"
- return error_template(423, "Quota exceeded, try again in a few hours")
- end
-
- if !tfa_code
- account_type = "google"
- captcha_type = "image"
-
- case tfa[8]
- when 6, 9
- prompt = "Google verification code"
- when 12
- prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
- when 15
- prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
- else
- prompt = "Google verification code"
- end
-
- tfa = nil
- captcha = nil
- return templated "user/login"
- end
-
- tl = challenge_results[1][2]
-
- request_type = tfa[8]
- case request_type
- when 6 # Authenticator app
- tfa_req = {
- user_hash, nil, 2, nil,
- {6, nil, nil, nil, nil,
- {tfa_code, false},
- },
- }.to_json
- when 9 # Voice or text message
- tfa_req = {
- user_hash, nil, 2, nil,
- {9, nil, nil, nil, nil, nil, nil, nil,
- {nil, tfa_code, false, 2},
- },
- }.to_json
- when 12 # Recovery email
- tfa_req = {
- user_hash, nil, 4, nil,
- {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
- {tfa_code},
- },
- }.to_json
- when 15 # Security question
- tfa_req = {
- user_hash, nil, 5, nil,
- {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
- {tfa_code},
- },
- }.to_json
- else
- return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.")
- end
-
- traceback << "Submitting challenge..."
-
- response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req))
- headers = response.cookies.add_request_headers(headers)
- challenge_results = JSON.parse(response.body[5..-1])
-
- if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") ||
- (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT")
- return error_template(401, "Invalid TFA code")
- end
-
- traceback << "done.
"
- end
-
- traceback << "Logging in..."
-
- location = URI.parse(challenge_results[0][-1][2].to_s)
- cookies = HTTP::Cookies.from_client_headers(headers)
-
- headers.delete("Content-Type")
- headers.delete("Google-Accounts-XSRF")
-
- loop do
- if !location || location.path == "/ManageAccount"
- break
- end
-
- # Occasionally there will be a second page after login confirming
- # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle.
-
- if location.path.starts_with? "/b/0/SmsAuthInterstitial"
- traceback << "Unhandled dialog /b/0/SmsAuthInterstitial."
- end
-
- login = client.get(location.request_target, headers)
-
- headers = login.cookies.add_request_headers(headers)
- location = login.headers["Location"]?.try { |u| URI.parse(u) }
- end
-
- cookies = HTTP::Cookies.from_client_headers(headers)
- sid = cookies["SID"]?.try &.value
- if !sid
- raise "Couldn't get SID."
- end
-
- user, sid = get_user(sid, headers)
-
- # We are now logged in
- traceback << "done.
"
-
- host = URI.parse(env.request.headers["Host"]).host
-
- cookies.each do |cookie|
- cookie.secure = Invidious::User::Cookies::SECURE
-
- if cookie.extension
- cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host)
- cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "")
- end
- env.response.cookies << cookie
- end
-
- if env.request.cookies["PREFS"]?
- user.preferences = env.get("preferences").as(Preferences)
- Invidious::Database::Users.update_preferences(user)
-
- cookie = env.request.cookies["PREFS"]
- cookie.expires = Time.utc(1990, 1, 1)
- env.response.cookies << cookie
- end
-
- env.redirect referer
- rescue ex
- traceback.rewind
- # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.")
- error_message = %(#{ex.message}
Traceback:
#{traceback.gets_to_end})
- return error_template(500, error_message)
- end
when "invidious"
- if !email
+ if email.nil? || email.empty?
return error_template(401, "User ID is a required field")
end
- if !password
+ if password.nil? || password.empty?
return error_template(401, "Password is a required field")
end
user = Invidious::Database::Users.select(email: email)
if user
- if !user.password
- return error_template(400, "Please sign in using 'Log in with Google'")
- end
-
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
Invidious::Database::SessionIDs.insert(sid, email)
@@ -356,36 +85,14 @@ module Invidious::Routes::Login
password = password.byte_slice(0, 55)
if CONFIG.captcha_enabled
- captcha_type = env.params.body["captcha_type"]?
answer = env.params.body["answer"]?
- change_type = env.params.body["change_type"]?
- if !captcha_type || change_type
- if change_type
- captcha_type = change_type
- end
- captcha_type ||= "image"
-
- account_type = "invidious"
- tfa = false
- prompt = ""
-
- if captcha_type == "image"
- captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
- else
- captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
- end
-
- return templated "user/login"
- end
+ account_type = "invidious"
+ captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
- answer ||= ""
- captcha_type ||= "image"
-
- case captcha_type
- when "image"
+ if answer
answer = answer.lstrip('0')
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
@@ -394,27 +101,8 @@ module Invidious::Routes::Login
rescue ex
return error_template(400, ex)
end
- else # "text"
- answer = Digest::MD5.hexdigest(answer.downcase.strip)
-
- if tokens.empty?
- return error_template(500, "Erroneous CAPTCHA")
- end
-
- found_valid_captcha = false
- error_exception = Exception.new
- tokens.each do |tok|
- begin
- validate_request(tok, answer, env.request, HMAC_KEY, locale)
- found_valid_captcha = true
- rescue ex
- error_exception = ex
- end
- end
-
- if !found_valid_captcha
- return error_template(500, error_exception)
- end
+ else
+ return templated "user/login"
end
end
@@ -481,11 +169,4 @@ module Invidious::Routes::Login
env.redirect referer
end
-
- def self.captcha(env)
- headers = HTTP::Headers{":authority" => "accounts.google.com"}
- response = YT_POOL.client &.get(env.request.resource, headers)
- env.response.headers["Content-Type"] = response.headers["Content-Type"]
- response.body
- end
end
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index d6bd9571..0b868755 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -40,7 +40,21 @@ module Invidious::Routes::Misc
def self.cross_instance_redirect(env)
referer = get_referer(env)
- instance_url = fetch_random_instance
+
+ instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
+ # Filter out the current instance
+ other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
+
+ if other_available_instances.empty?
+ # If the current instance is the only one, use the redirect URL as fallback
+ instance_url = "redirect.invidious.io"
+ else
+ # Select other random instance
+ # Sample returns an array
+ # Instances are packaged as {region, domain} in the instance list
+ instance_url = other_available_instances.sample(1)[0][1]
+ end
+
env.redirect "https://#{instance_url}#{referer}"
end
end
diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr
index 272a3dc7..8922b740 100644
--- a/src/invidious/routes/notifications.cr
+++ b/src/invidious/routes/notifications.cr
@@ -24,50 +24,6 @@ module Invidious::Routes::Notifications
user = user.as(User)
- if !user.password
- channel_req = {} of String => String
-
- channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true"
- channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || ""
- channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true"
-
- channel_req.reject! { |k, v| v != "true" && v != "false" }
-
- headers = HTTP::Headers.new
- headers["Cookie"] = env.request.headers["Cookie"]
-
- html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
-
- cookies = HTTP::Cookies.from_client_headers(headers)
- html.cookies.each do |cookie|
- if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
- if cookies[cookie.name]?
- cookies[cookie.name] = cookie
- else
- cookies << cookie
- end
- end
- end
- headers = cookies.add_request_headers(headers)
-
- if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
- session_token = match["session_token"]
- else
- return env.redirect referer
- end
-
- headers["content-type"] = "application/x-www-form-urlencoded"
- channel_req["session_token"] = session_token
-
- subs = XML.parse_html(html.body)
- subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel|
- channel_id = channel.content.lstrip("/channel/").not_nil!
- channel_req["channel_id"] = channel_id
-
- YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
- end
- end
-
if redirect
env.redirect referer
else
diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr
index dbeb4f97..f2213da4 100644
--- a/src/invidious/routes/playlists.cr
+++ b/src/invidious/routes/playlists.cr
@@ -66,7 +66,13 @@ module Invidious::Routes::Playlists
user = user.as(User)
playlist_id = env.params.query["list"]
- playlist = get_playlist(playlist_id)
+ begin
+ playlist = get_playlist(playlist_id)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
+ rescue ex
+ return error_template(500, ex)
+ end
subscribe_playlist(user, playlist)
env.redirect "/playlist?list=#{playlist.id}"
@@ -157,13 +163,20 @@ module Invidious::Routes::Playlists
end
begin
- videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
+ items = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
- videos = [] of PlaylistVideo
+ items = [] of PlaylistVideo
end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY)
+ # Pagination
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/playlist?list=#{playlist.id}",
+ current_page: page,
+ show_next: (items.size == 100)
+ )
+
templated "edit_playlist"
end
@@ -212,7 +225,10 @@ module Invidious::Routes::Playlists
end
def self.add_playlist_items_page(env)
- locale = env.get("preferences").as(Preferences).locale
+ prefs = env.get("preferences").as(Preferences)
+ locale = prefs.locale
+
+ region = env.params.query["region"]? || prefs.region
user = env.get? "user"
sid = env.get? "sid"
@@ -236,18 +252,21 @@ module Invidious::Routes::Playlists
return env.redirect referer
end
- query = env.params.query["q"]?
- if query
- begin
- search_query, items, operators = process_search_query(query, page, user, region: nil)
- videos = items.select(SearchVideo).map(&.as(SearchVideo))
- rescue ex
- videos = [] of SearchVideo
- end
- else
- videos = [] of SearchVideo
+ begin
+ query = Invidious::Search::Query.new(env.params.query, :playlist, region)
+ items = query.process.select(SearchVideo).map(&.as(SearchVideo))
+ rescue ex
+ items = [] of SearchVideo
end
+ # Pagination
+ query_encoded = URI.encode_www_form(query.try &.text || "", space_to_plus: true)
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/add_playlist_items?list=#{playlist.id}&q=#{query_encoded}",
+ current_page: page,
+ show_next: (items.size >= 20)
+ )
+
env.set "add_playlist_items", plid
templated "add_playlist_items"
end
@@ -285,27 +304,12 @@ module Invidious::Routes::Playlists
end
end
- if env.params.query["action_create_playlist"]?
- action = "action_create_playlist"
- elsif env.params.query["action_delete_playlist"]?
- action = "action_delete_playlist"
- elsif env.params.query["action_edit_playlist"]?
- action = "action_edit_playlist"
- elsif env.params.query["action_add_video"]?
- action = "action_add_video"
- video_id = env.params.query["video_id"]
- elsif env.params.query["action_remove_video"]?
- action = "action_remove_video"
- elsif env.params.query["action_move_video_before"]?
- action = "action_move_video_before"
- else
- return env.redirect referer
- end
-
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
raise "Invalid user" if playlist.author != user.email
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
if redirect
return error_template(400, ex)
@@ -314,21 +318,13 @@ module Invidious::Routes::Playlists
end
end
- if !user.password
- # TODO: Playlist stub, sync with YouTube for Google accounts
- # playlist_ajax(playlist_id, action, env.request.headers)
- end
- email = user.email
-
- case action
- when "action_edit_playlist"
- # TODO: Playlist stub
- when "action_add_video"
- if playlist.index.size >= 500
+ case action = env.params.query["action"]?
+ when "add_video"
+ if playlist.index.size >= CONFIG.playlist_length_limit
if redirect
- return error_template(400, "Playlist cannot have more than 500 videos")
+ return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
else
- return error_json(400, "Playlist cannot have more than 500 videos")
+ return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
end
@@ -336,6 +332,8 @@ module Invidious::Routes::Playlists
begin
video = get_video(video_id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
if redirect
return error_template(500, ex)
@@ -358,12 +356,14 @@ module Invidious::Routes::Playlists
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
- when "action_remove_video"
+ when "remove_video"
index = env.params.query["set_video_id"]
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(playlist_id, index)
- when "action_move_video_before"
+ when "move_video_before"
# TODO: Playlist stub
+ when nil
+ return error_json(400, "Missing action")
else
return error_json(400, "Unsupported action #{action}")
end
@@ -396,12 +396,19 @@ module Invidious::Routes::Playlists
begin
playlist = get_playlist(plid)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- page_count = (playlist.video_count / 100).to_i
- page_count += 1 if (playlist.video_count % 100) > 0
+ if playlist.is_a? InvidiousPlaylist
+ page_count = (playlist.video_count / 100).to_i
+ page_count += 1 if (playlist.video_count % 100) > 0
+ else
+ page_count = (playlist.video_count / 200).to_i
+ page_count += 1 if (playlist.video_count % 200) > 0
+ end
if page > page_count
return env.redirect "/playlist?list=#{plid}&page=#{page_count}"
@@ -412,7 +419,11 @@ module Invidious::Routes::Playlists
end
begin
- videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
+ if playlist.is_a? InvidiousPlaylist
+ items = get_playlist_videos(playlist, offset: (page - 1) * 100)
+ else
+ items = get_playlist_videos(playlist, offset: (page - 1) * 200)
+ end
rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}")
end
@@ -421,6 +432,13 @@ module Invidious::Routes::Playlists
env.set "remove_playlist_items", plid
end
+ # Pagination
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/playlist?list=#{playlist.id}",
+ current_page: page,
+ show_next: (page_count != 1 && page < page_count)
+ )
+
templated "playlist"
end
diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr
index 570cba69..39ca77c0 100644
--- a/src/invidious/routes/preferences.cr
+++ b/src/invidious/routes/preferences.cr
@@ -27,6 +27,10 @@ module Invidious::Routes::PreferencesRoute
annotations_subscribed ||= "off"
annotations_subscribed = annotations_subscribed == "on"
+ preload = env.params.body["preload"]?.try &.as(String)
+ preload ||= "off"
+ preload = preload == "on"
+
autoplay = env.params.body["autoplay"]?.try &.as(String)
autoplay ||= "off"
autoplay = autoplay == "on"
@@ -144,6 +148,7 @@ module Invidious::Routes::PreferencesRoute
preferences = Preferences.from_json({
annotations: annotations,
annotations_subscribed: annotations_subscribed,
+ preload: preload,
autoplay: autoplay,
captions: captions,
comments: comments,
@@ -214,7 +219,7 @@ module Invidious::Routes::PreferencesRoute
statistics_enabled ||= "off"
CONFIG.statistics_enabled = statistics_enabled == "on"
- CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
+ CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
File.write("config/config.yml", CONFIG.to_yaml)
end
@@ -310,6 +315,24 @@ module Invidious::Routes::PreferencesRoute
response: error_template(415, "Invalid subscription file uploaded")
)
end
+ when "import_youtube_pl"
+ filename = part.filename || ""
+ success = Invidious::User::Import.from_youtube_pl(user, body, filename, type)
+
+ if !success
+ haltf(env, status_code: 415,
+ response: error_template(415, "Invalid playlist file uploaded")
+ )
+ end
+ when "import_youtube_wh"
+ filename = part.filename || ""
+ success = Invidious::User::Import.from_youtube_wh(user, body, filename, type)
+
+ if !success
+ haltf(env, status_code: 415,
+ response: error_template(415, "Invalid watch history file uploaded")
+ )
+ end
when "import_freetube"
Invidious::User::Import.from_freetube(user, body)
when "import_newpipe_subscriptions"
diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr
index 3f4c7e5e..b195c7b3 100644
--- a/src/invidious/routes/search.cr
+++ b/src/invidious/routes/search.cr
@@ -37,38 +37,87 @@ module Invidious::Routes::Search
end
def self.search(env)
- locale = env.get("preferences").as(Preferences).locale
- region = env.params.query["region"]?
+ prefs = env.get("preferences").as(Preferences)
+ locale = prefs.locale
- query = env.params.query["search_query"]?
- query ||= env.params.query["q"]?
+ region = env.params.query["region"]? || prefs.region
- if !query || query.empty?
+ query = Invidious::Search::Query.new(env.params.query, :regular, region)
+
+ if query.empty?
# Display the full page search box implemented in #1977
env.set "search", ""
templated "search_homepage", navbar_search: false
else
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
-
user = env.get? "user"
+ # An URL was copy/pasted in the search box.
+ # Redirect the user to the appropriate page.
+ if query.url?
+ return env.redirect UrlSanitizer.process(query.text).to_s
+ end
+
begin
- search_query, videos, operators = process_search_query(query, page, user, region: region)
+ if user
+ items = query.process(user.as(User))
+ else
+ items = query.process
+ end
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex
return error_template(500, ex)
end
- operator_hash = {} of String => String
- operators.each do |operator|
- key, value = operator.downcase.split(":")
- operator_hash[key] = value
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ # Pagination
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/search?#{query.to_http_params}",
+ current_page: query.page,
+ show_next: (items.size >= 20)
+ )
+
+ if query.type == Invidious::Search::Query::Type::Channel
+ env.set "search", "channel:#{query.channel} #{query.text}"
+ else
+ env.set "search", query.text
end
- env.set "search", query
templated "search"
end
end
+
+ def self.hashtag(env : HTTP::Server::Context)
+ locale = env.get("preferences").as(Preferences).locale
+
+ hashtag = env.params.url["hashtag"]?
+ if hashtag.nil? || hashtag.empty?
+ return error_template(400, "Invalid request")
+ end
+
+ page = env.params.query["page"]?
+ if page.nil?
+ page = 1
+ else
+ page = Math.max(1, page.to_i)
+ env.params.query.delete_all("page")
+ end
+
+ begin
+ items = Invidious::Hashtag.fetch(hashtag, page)
+ rescue ex
+ return error_template(500, ex)
+ end
+
+ # Pagination
+ hashtag_encoded = URI.encode_www_form(hashtag, space_to_plus: false)
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/hashtag/#{hashtag_encoded}",
+ current_page: page,
+ show_next: (items.size >= 60)
+ )
+
+ templated "hashtag"
+ end
end
diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr
index 7b1fa876..1de655d2 100644
--- a/src/invidious/routes/subscriptions.cr
+++ b/src/invidious/routes/subscriptions.cr
@@ -32,29 +32,16 @@ module Invidious::Routes::Subscriptions
end
end
- if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1
- action = "action_create_subscription_to_channel"
- elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1
- action = "action_remove_subscriptions"
- else
- return env.redirect referer
- end
-
channel_id = env.params.query["c"]?
channel_id ||= ""
- if !user.password
- # Sync subscriptions with YouTube
- subscribe_ajax(channel_id, action, env.request.headers)
- end
-
- case action
- when "action_create_subscription_to_channel"
+ case action = env.params.query["action"]?
+ when "create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
- when "action_remove_subscriptions"
+ when "remove_subscriptions"
Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else
return error_json(400, "Unsupported action #{action}")
@@ -82,14 +69,6 @@ module Invidious::Routes::Subscriptions
user = user.as(User)
sid = sid.as(String)
- if !user.password
- # Refresh account
- headers = HTTP::Headers.new
- headers["Cookie"] = env.request.headers["Cookie"]
-
- user, sid = get_user(sid, headers)
- end
-
action_takeout = env.params.query["action_takeout"]?.try &.to_i?
action_takeout ||= 0
action_takeout = action_takeout == 1
@@ -104,33 +83,8 @@ module Invidious::Routes::Subscriptions
if format == "json"
env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment"
- playlists = Invidious::Database::Playlists.select_like_iv(user.email)
- return JSON.build do |json|
- json.object do
- json.field "subscriptions", user.subscriptions
- json.field "watch_history", user.watched
- json.field "preferences", user.preferences
- json.field "playlists" do
- json.array do
- playlists.each do |playlist|
- json.object do
- json.field "title", playlist.title
- json.field "description", html_to_content(playlist.description_html)
- json.field "privacy", playlist.privacy.to_s
- json.field "videos" do
- json.array do
- Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id|
- json.string video_id
- end
- end
- end
- end
- end
- end
- end
- end
- end
+ return Invidious::User::Export.to_invidious(user)
else
env.response.content_type = "application/xml"
env.response.headers["content-disposition"] = "attachment"
diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index 3a92ef96..083087a9 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -21,7 +21,7 @@ module Invidious::Routes::VideoPlayback
end
# Sanity check, to avoid being used as an open proxy
- if !host.matches?(/[\w-]+.googlevideo.com/)
+ if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/)
return error_template(400, "Invalid \"host\" parameter.")
end
@@ -35,7 +35,15 @@ module Invidious::Routes::VideoPlayback
end
end
- client = make_client(URI.parse(host), region)
+ # See: https://github.com/iv-org/invidious/issues/3302
+ range_header = env.request.headers["Range"]?
+ sq = query_params["sq"]?
+ if range_header.nil? && sq.nil?
+ range_for_head = query_params["range"]? || "0-640"
+ headers["Range"] = "bytes=#{range_for_head}"
+ end
+
+ client = make_client(URI.parse(host), region, force_resolve: true)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
@@ -50,7 +58,7 @@ module Invidious::Routes::VideoPlayback
if new_host != host
host = new_host
client.close
- client = make_client(URI.parse(new_host), region)
+ client = make_client(URI.parse(new_host), region, force_resolve: true)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
@@ -64,15 +72,23 @@ module Invidious::Routes::VideoPlayback
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
- client = make_client(URI.parse(host), region)
+ client = make_client(URI.parse(host), region, force_resolve: true)
rescue ex
error = ex.message
end
end
+ # Remove the Range header added previously.
+ headers.delete("Range") if range_header.nil?
+
+ playback_statistics = get_playback_statistic()
+ playback_statistics["totalRequests"] += 1
+
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
+ else
+ playback_statistics["successfulRequests"] += 1
end
if url.includes? "&file=seg.ts"
@@ -91,14 +107,8 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- location = URI.parse(location)
- location = "#{location.request_target}&host=#{location.host}"
-
- if region
- location += "®ion=#{region}"
- end
-
- return env.redirect location
+ url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
+ return env.redirect url
end
IO.copy(resp.body_io, env.response)
@@ -122,7 +132,7 @@ module Invidious::Routes::VideoPlayback
end
# TODO: Record bytes written so we can restart after a chunk fails
- while true
+ loop do
if !range_end && content_length
range_end = content_length
end
@@ -155,10 +165,13 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- location = URI.parse(location)
- location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
+ url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
- env.redirect location
+ if title = query_params["title"]?
+ url = "#{url}&title=#{URI.encode_www_form(title)}"
+ end
+
+ env.redirect url
break
end
@@ -187,7 +200,7 @@ module Invidious::Routes::VideoPlayback
break
else
client.close
- client = make_client(URI.parse(host), region)
+ client = make_client(URI.parse(host), region, force_resolve: true)
end
end
@@ -244,6 +257,11 @@ module Invidious::Routes::VideoPlayback
# YouTube /videoplayback links expire after 6 hours,
# so we have a mechanism here to redirect to the latest version
def self.latest_version(env)
+ if CONFIG.invidious_companion.present?
+ invidious_companion = CONFIG.invidious_companion.sample
+ return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
+ end
+
id = env.params.query["id"]?
itag = env.params.query["itag"]?.try &.to_i?
@@ -252,7 +270,7 @@ module Invidious::Routes::VideoPlayback
return error_template(400, "Invalid video ID")
end
- if itag.nil? || itag <= 0 || itag >= 1000
+ if !itag.nil? && (itag <= 0 || itag >= 1000)
return error_template(400, "Invalid itag")
end
@@ -265,9 +283,19 @@ module Invidious::Routes::VideoPlayback
return error_template(403, "Administrator has disabled this endpoint.")
end
- video = get_video(id, region: region)
+ begin
+ video = get_video(id, region: region)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
+ rescue ex
+ return error_template(500, ex)
+ end
- fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
+ if itag.nil?
+ fmt = video.fmt_stream[-1]?
+ else
+ fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
+ end
url = fmt.try &.["url"]?.try &.as_s
if !url
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
index 867ffa6a..e777b3f1 100644
--- a/src/invidious/routes/watch.cr
+++ b/src/invidious/routes/watch.cr
@@ -30,14 +30,6 @@ module Invidious::Routes::Watch
return env.redirect "/"
end
- embed_link = "/embed/#{id}"
- if env.params.query.size > 1
- embed_params = HTTP::Params.parse(env.params.query.to_s)
- embed_params.delete_all("v")
- embed_link += "?"
- embed_link += embed_params.to_s
- end
-
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(env.params.query, plid, id)
@@ -61,8 +53,9 @@ module Invidious::Routes::Watch
begin
video = get_video(id, region: params.region)
- rescue ex : VideoRedirect
- return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex : NotFoundException
+ LOGGER.error("get_video not found: #{id} : #{ex.message}")
+ return error_template(404, ex)
rescue ex
LOGGER.error("get_video: #{id} : #{ex.message}")
return error_template(500, ex)
@@ -75,11 +68,11 @@ module Invidious::Routes::Watch
end
env.params.query.delete_all("iv_load_policy")
- if watched && preferences.watch_history && !watched.includes? id
+ if watched && preferences.watch_history
Invidious::Database::Users.mark_watched(user.as(User), id)
end
- if notifications && notifications.includes? id
+ if CONFIG.enable_user_notifications && notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
@@ -94,31 +87,31 @@ module Invidious::Routes::Watch
if source == "youtube"
begin
- comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
rescue ex
if preferences.comments[1] == "reddit"
- comments, reddit_thread = fetch_reddit_comments(id)
- comment_html = template_reddit_comments(comments, locale)
+ comments, reddit_thread = Comments.fetch_reddit(id)
+ comment_html = Frontend::Comments.template_reddit(comments, locale)
- comment_html = fill_links(comment_html, "https", "www.reddit.com")
- comment_html = replace_links(comment_html)
+ comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = Comments.replace_links(comment_html)
end
end
elsif source == "reddit"
begin
- comments, reddit_thread = fetch_reddit_comments(id)
- comment_html = template_reddit_comments(comments, locale)
+ comments, reddit_thread = Comments.fetch_reddit(id)
+ comment_html = Frontend::Comments.template_reddit(comments, locale)
- comment_html = fill_links(comment_html, "https", "www.reddit.com")
- comment_html = replace_links(comment_html)
+ comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = Comments.replace_links(comment_html)
rescue ex
if preferences.comments[1] == "youtube"
- comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
end
end
else
- comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
comment_html ||= ""
@@ -128,10 +121,12 @@ module Invidious::Routes::Watch
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
end
+ # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -197,6 +192,14 @@ module Invidious::Routes::Watch
captions: video.captions
)
+ if CONFIG.invidious_companion.present?
+ invidious_companion = CONFIG.invidious_companion.sample
+ env.response.headers["Content-Security-Policy"] =
+ env.response.headers["Content-Security-Policy"]
+ .gsub("media-src", "media-src #{invidious_companion.public_url}")
+ .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
+ end
+
templated "watch"
end
@@ -248,20 +251,10 @@ module Invidious::Routes::Watch
end
end
- if env.params.query["action_mark_watched"]?
- action = "action_mark_watched"
- elsif env.params.query["action_mark_unwatched"]?
- action = "action_mark_unwatched"
- else
- return env.redirect referer
- end
-
- case action
- when "action_mark_watched"
- if !user.watched.includes? id
- Invidious::Database::Users.mark_watched(user, id)
- end
- when "action_mark_unwatched"
+ case action = env.params.query["action"]?
+ when "mark_watched"
+ Invidious::Database::Users.mark_watched(user, id)
+ when "mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
else
return error_json(400, "Unsupported action #{action}")
@@ -284,6 +277,12 @@ module Invidious::Routes::Watch
return error_template(400, "Invalid clip ID") if response["error"]?
if video_id = response.dig?("endpoint", "watchEndpoint", "videoId")
+ if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
+ start_time, end_time, _ = parse_clip_parameters(params)
+ env.params.query["start"] = start_time.to_s if start_time != nil
+ env.params.query["end"] = end_time.to_s if end_time != nil
+ end
+
return env.redirect "/watch?v=#{video_id}&#{env.params.query}"
else
return error_template(404, "The requested clip doesn't exist")
@@ -294,6 +293,9 @@ module Invidious::Routes::Watch
if CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.")
end
+ if CONFIG.invidious_companion.present?
+ return error_template(403, "Downloads should be routed through Companion when present")
+ end
title = env.params.body["title"]? || ""
video_id = env.params.body["id"]? || ""
@@ -306,27 +308,27 @@ module Invidious::Routes::Watch
download_widget = JSON.parse(selection)
extension = download_widget["ext"].as_s
- filename = "#{video_id}-#{title}.#{extension}"
+ filename = "#{title}-#{video_id}.#{extension}"
- # Pass form parameters as URL parameters for the handlers of both
- # /latest_version and /api/v1/captions. This avoids an un-necessary
- # redirect and duplicated (and hazardous) sanity checks.
- env.params.query["id"] = video_id
- env.params.query["title"] = filename
-
- # Delete the useless ones
+ # Delete the now useless URL parameters
env.params.body.delete("id")
env.params.body.delete("title")
env.params.body.delete("download_widget")
+ # Pass form parameters as URL parameters for the handlers of both
+ # /latest_version and /api/v1/captions. This avoids an un-necessary
+ # redirect and duplicated (and hazardous) sanity checks.
if label = download_widget["label"]?
# URL params specific to /api/v1/captions/:id
- env.params.query["label"] = URI.encode_www_form(label.as_s, space_to_plus: false)
+ env.params.url["id"] = video_id
+ env.params.query["title"] = filename
+ env.params.query["label"] = URI.decode_www_form(label.as_s)
return Invidious::Routes::API::V1::Videos.captions(env)
- elsif itag = download_widget["itag"]?.try &.as_i
+ elsif itag = download_widget["itag"]?.try &.as_i.to_s
# URL params specific to /latest_version
- env.params.query["itag"] = itag.to_s
+ env.params.query["id"] = video_id
+ env.params.query["title"] = filename
env.params.query["local"] = "true"
return Invidious::Routes::VideoPlayback.latest_version(env)
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index bd72c577..46b71f1f 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -1,130 +1,319 @@
module Invidious::Routing
- {% for http_method in {"get", "post", "delete", "options", "patch", "put", "head"} %}
+ extend self
+
+ {% for http_method in {"get", "post", "delete", "options", "patch", "put"} %}
macro {{http_method.id}}(path, controller, method = :handle)
- {{http_method.id}} \{{ path }} do |env|
+ unless Kemal::Utils.path_starts_with_slash?(\{{path}})
+ raise Kemal::Exceptions::InvalidPathStartException.new({{http_method}}, \{{path}})
+ end
+
+ Kemal::RouteHandler::INSTANCE.add_route({{http_method.upcase}}, \{{path}}) do |env|
\{{ controller }}.\{{ method.id }}(env)
end
end
{% end %}
-end
-
-macro define_user_routes
- # User login/out
- Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page
- Invidious::Routing.post "/login", Invidious::Routes::Login, :login
- Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout
- Invidious::Routing.get "/Captcha", Invidious::Routes::Login, :captcha
-
- # User preferences
- Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show
- Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update
- Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme
- Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control
- Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control
-
- # User account management
- Invidious::Routing.get "/change_password", Invidious::Routes::Account, :get_change_password
- Invidious::Routing.post "/change_password", Invidious::Routes::Account, :post_change_password
- Invidious::Routing.get "/delete_account", Invidious::Routes::Account, :get_delete
- Invidious::Routing.post "/delete_account", Invidious::Routes::Account, :post_delete
- Invidious::Routing.get "/clear_watch_history", Invidious::Routes::Account, :get_clear_history
- Invidious::Routing.post "/clear_watch_history", Invidious::Routes::Account, :post_clear_history
- Invidious::Routing.get "/authorize_token", Invidious::Routes::Account, :get_authorize_token
- Invidious::Routing.post "/authorize_token", Invidious::Routes::Account, :post_authorize_token
- Invidious::Routing.get "/token_manager", Invidious::Routes::Account, :token_manager
- Invidious::Routing.post "/token_ajax", Invidious::Routes::Account, :token_ajax
-end
-
-macro define_v1_api_routes
- {{namespace = Invidious::Routes::API::V1}}
- # Videos
- Invidious::Routing.get "/api/v1/videos/:id", {{namespace}}::Videos, :videos
- Invidious::Routing.get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards
- Invidious::Routing.get "/api/v1/captions/:id", {{namespace}}::Videos, :captions
- Invidious::Routing.get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
- Invidious::Routing.get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
-
- # Feeds
- Invidious::Routing.get "/api/v1/trending", {{namespace}}::Feeds, :trending
- Invidious::Routing.get "/api/v1/popular", {{namespace}}::Feeds, :popular
-
- # Channels
- Invidious::Routing.get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
- {% for route in {"videos", "latest", "playlists", "community", "search"} %}
- Invidious::Routing.get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
- Invidious::Routing.get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
- {% end %}
-
- # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community
- Invidious::Routing.get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect
- Invidious::Routing.get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect
-
-
- # Search
- Invidious::Routing.get "/api/v1/search", {{namespace}}::Search, :search
- Invidious::Routing.get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions
-
- # Authenticated
-
- # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr
- #
- # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
- # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
-
- Invidious::Routing.get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
- Invidious::Routing.post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
-
- Invidious::Routing.get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed
-
- Invidious::Routing.get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions
- Invidious::Routing.post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel
- Invidious::Routing.delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel
-
-
- Invidious::Routing.get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists
- Invidious::Routing.post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist
- Invidious::Routing.patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute
- Invidious::Routing.delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist
-
-
- Invidious::Routing.post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist
- Invidious::Routing.delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist
-
- Invidious::Routing.get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens
- Invidious::Routing.post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token
- Invidious::Routing.post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token
-
- Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
- Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
-
- # Misc
- Invidious::Routing.get "/api/v1/stats", {{namespace}}::Misc, :stats
- Invidious::Routing.get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist
- Invidious::Routing.get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist
- Invidious::Routing.get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes
-end
-
-macro define_api_manifest_routes
- Invidious::Routing.get "/api/manifest/dash/id/:id", Invidious::Routes::API::Manifest, :get_dash_video_id
-
- Invidious::Routing.get "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :get_dash_video_playback
- Invidious::Routing.get "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :get_dash_video_playback_greedy
-
- Invidious::Routing.options "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :options_dash_video_playback
- Invidious::Routing.options "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :options_dash_video_playback
-
- Invidious::Routing.get "/api/manifest/hls_playlist/*", Invidious::Routes::API::Manifest, :get_hls_playlist
- Invidious::Routing.get "/api/manifest/hls_variant/*", Invidious::Routes::API::Manifest, :get_hls_variant
-end
-
-macro define_video_playback_routes
- Invidious::Routing.get "/videoplayback", Invidious::Routes::VideoPlayback, :get_video_playback
- Invidious::Routing.get "/videoplayback/*", Invidious::Routes::VideoPlayback, :get_video_playback_greedy
-
- Invidious::Routing.options "/videoplayback", Invidious::Routes::VideoPlayback, :options_video_playback
- Invidious::Routing.options "/videoplayback/*", Invidious::Routes::VideoPlayback, :options_video_playback
-
- Invidious::Routing.get "/latest_version", Invidious::Routes::VideoPlayback, :latest_version
+
+ def register_all
+ {% unless flag?(:api_only) %}
+ get "/", Routes::Misc, :home
+ get "/privacy", Routes::Misc, :privacy
+ get "/licenses", Routes::Misc, :licenses
+ get "/redirect", Routes::Misc, :cross_instance_redirect
+
+ self.register_channel_routes
+ self.register_watch_routes
+
+ self.register_iv_playlist_routes
+ self.register_yt_playlist_routes
+
+ self.register_search_routes
+
+ self.register_user_routes
+ self.register_feed_routes
+
+ # Support push notifications via PubSubHubbub
+ get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get
+ post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post
+
+ if CONFIG.enable_user_notifications
+ get "/modify_notifications", Routes::Notifications, :modify
+ end
+ {% end %}
+
+ self.register_image_routes
+ self.register_api_v1_routes
+ self.register_api_manifest_routes
+ self.register_video_playback_routes
+ end
+
+ # -------------------
+ # Invidious routes
+ # -------------------
+
+ def register_user_routes
+ # User login/out
+ get "/login", Routes::Login, :login_page
+ post "/login", Routes::Login, :login
+ post "/signout", Routes::Login, :signout
+
+ # User preferences
+ get "/preferences", Routes::PreferencesRoute, :show
+ post "/preferences", Routes::PreferencesRoute, :update
+ get "/toggle_theme", Routes::PreferencesRoute, :toggle_theme
+ get "/data_control", Routes::PreferencesRoute, :data_control
+ post "/data_control", Routes::PreferencesRoute, :update_data_control
+
+ # User account management
+ get "/change_password", Routes::Account, :get_change_password
+ post "/change_password", Routes::Account, :post_change_password
+ get "/delete_account", Routes::Account, :get_delete
+ post "/delete_account", Routes::Account, :post_delete
+ get "/clear_watch_history", Routes::Account, :get_clear_history
+ post "/clear_watch_history", Routes::Account, :post_clear_history
+ get "/authorize_token", Routes::Account, :get_authorize_token
+ post "/authorize_token", Routes::Account, :post_authorize_token
+ get "/token_manager", Routes::Account, :token_manager
+ post "/token_ajax", Routes::Account, :token_ajax
+ post "/subscription_ajax", Routes::Subscriptions, :toggle_subscription
+ get "/subscription_manager", Routes::Subscriptions, :subscription_manager
+ end
+
+ def register_iv_playlist_routes
+ get "/create_playlist", Routes::Playlists, :new
+ post "/create_playlist", Routes::Playlists, :create
+ get "/subscribe_playlist", Routes::Playlists, :subscribe
+ get "/delete_playlist", Routes::Playlists, :delete_page
+ post "/delete_playlist", Routes::Playlists, :delete
+ get "/edit_playlist", Routes::Playlists, :edit
+ post "/edit_playlist", Routes::Playlists, :update
+ get "/add_playlist_items", Routes::Playlists, :add_playlist_items_page
+ post "/playlist_ajax", Routes::Playlists, :playlist_ajax
+ end
+
+ def register_feed_routes
+ # Feeds
+ get "/view_all_playlists", Routes::Feeds, :view_all_playlists_redirect
+ get "/feed/playlists", Routes::Feeds, :playlists
+ get "/feed/popular", Routes::Feeds, :popular
+ get "/feed/trending", Routes::Feeds, :trending
+ get "/feed/subscriptions", Routes::Feeds, :subscriptions
+ get "/feed/history", Routes::Feeds, :history
+
+ # RSS Feeds
+ get "/feed/channel/:ucid", Routes::Feeds, :rss_channel
+ get "/feed/private", Routes::Feeds, :rss_private
+ get "/feed/playlist/:plid", Routes::Feeds, :rss_playlist
+ get "/feeds/videos.xml", Routes::Feeds, :rss_videos
+ end
+
+ # -------------------
+ # Youtube routes
+ # -------------------
+
+ def register_channel_routes
+ get "/channel/:ucid", Routes::Channels, :home
+ get "/channel/:ucid/home", Routes::Channels, :home
+ get "/channel/:ucid/videos", Routes::Channels, :videos
+ get "/channel/:ucid/shorts", Routes::Channels, :shorts
+ get "/channel/:ucid/streams", Routes::Channels, :streams
+ get "/channel/:ucid/podcasts", Routes::Channels, :podcasts
+ get "/channel/:ucid/releases", Routes::Channels, :releases
+ get "/channel/:ucid/courses", Routes::Channels, :courses
+ get "/channel/:ucid/playlists", Routes::Channels, :playlists
+ get "/channel/:ucid/community", Routes::Channels, :community
+ get "/channel/:ucid/posts", Routes::Channels, :community
+ get "/channel/:ucid/channels", Routes::Channels, :channels
+ get "/channel/:ucid/about", Routes::Channels, :about
+
+ get "/channel/:ucid/live", Routes::Channels, :live
+ get "/user/:user/live", Routes::Channels, :live
+ get "/c/:user/live", Routes::Channels, :live
+ get "/post/:id", Routes::Channels, :post
+
+ # Channel catch-all, to redirect future routes to the channel's home
+ # NOTE: defined last in order to be processed after the other routes
+ get "/channel/:ucid/*", Routes::Channels, :redirect_home
+
+ # /c/LinusTechTips
+ get "/c/:user", Routes::Channels, :brand_redirect
+ get "/c/:user/:tab", Routes::Channels, :brand_redirect
+
+ # /user/linustechtips (Not always the same as /c/)
+ get "/user/:user", Routes::Channels, :brand_redirect
+ get "/user/:user/:tab", Routes::Channels, :brand_redirect
+
+ # /@LinusTechTips (Handle)
+ get "/@:user", Routes::Channels, :brand_redirect
+ get "/@:user/:tab", Routes::Channels, :brand_redirect
+
+ # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow
+ get "/attribution_link", Routes::Channels, :brand_redirect
+ get "/attribution_link/:tab", Routes::Channels, :brand_redirect
+
+ # /profile?user=linustechtips
+ get "/profile", Routes::Channels, :profile
+ get "/profile/*", Routes::Channels, :profile
+ end
+
+ def register_watch_routes
+ get "/watch", Routes::Watch, :handle
+ post "/watch_ajax", Routes::Watch, :mark_watched
+ get "/watch/:id", Routes::Watch, :redirect
+ get "/live/:id", Routes::Watch, :redirect
+ get "/shorts/:id", Routes::Watch, :redirect
+ get "/clip/:clip", Routes::Watch, :clip
+ get "/w/:id", Routes::Watch, :redirect
+ get "/v/:id", Routes::Watch, :redirect
+ get "/e/:id", Routes::Watch, :redirect
+
+ post "/download", Routes::Watch, :download
+
+ get "/embed/", Routes::Embed, :redirect
+ get "/embed/:id", Routes::Embed, :show
+ end
+
+ def register_yt_playlist_routes
+ get "/playlist", Routes::Playlists, :show
+ get "/mix", Routes::Playlists, :mix
+ get "/watch_videos", Routes::Playlists, :watch_videos
+ end
+
+ def register_search_routes
+ get "/opensearch.xml", Routes::Search, :opensearch
+ get "/results", Routes::Search, :results
+ get "/search", Routes::Search, :search
+ get "/hashtag/:hashtag", Routes::Search, :hashtag
+ end
+
+ # -------------------
+ # Media proxy routes
+ # -------------------
+
+ def register_api_manifest_routes
+ get "/api/manifest/dash/id/:id", Routes::API::Manifest, :get_dash_video_id
+
+ get "/api/manifest/dash/id/videoplayback", Routes::API::Manifest, :get_dash_video_playback
+ get "/api/manifest/dash/id/videoplayback/*", Routes::API::Manifest, :get_dash_video_playback_greedy
+
+ options "/api/manifest/dash/id/videoplayback", Routes::API::Manifest, :options_dash_video_playback
+ options "/api/manifest/dash/id/videoplayback/*", Routes::API::Manifest, :options_dash_video_playback
+
+ get "/api/manifest/hls_playlist/*", Routes::API::Manifest, :get_hls_playlist
+ get "/api/manifest/hls_variant/*", Routes::API::Manifest, :get_hls_variant
+ end
+
+ def register_video_playback_routes
+ get "/videoplayback", Routes::VideoPlayback, :get_video_playback
+ get "/videoplayback/*", Routes::VideoPlayback, :get_video_playback_greedy
+
+ options "/videoplayback", Routes::VideoPlayback, :options_video_playback
+ options "/videoplayback/*", Routes::VideoPlayback, :options_video_playback
+
+ get "/latest_version", Routes::VideoPlayback, :latest_version
+ end
+
+ def register_image_routes
+ get "/ggpht/*", Routes::Images, :ggpht
+ options "/sb/:authority/:id/:storyboard/:index", Routes::Images, :options_storyboard
+ get "/sb/:authority/:id/:storyboard/:index", Routes::Images, :get_storyboard
+ get "/s_p/:id/:name", Routes::Images, :s_p_image
+ get "/yts/img/:name", Routes::Images, :yts_image
+ get "/vi/:id/:name", Routes::Images, :thumbnails
+ end
+
+ # -------------------
+ # API routes
+ # -------------------
+
+ def register_api_v1_routes
+ {% begin %}
+ {{namespace = Routes::API::V1}}
+
+ # Videos
+ get "/api/v1/videos/:id", {{namespace}}::Videos, :videos
+ get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards
+ get "/api/v1/captions/:id", {{namespace}}::Videos, :captions
+ get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
+ get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
+ get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
+ get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts
+
+ # Feeds
+ get "/api/v1/trending", {{namespace}}::Feeds, :trending
+ get "/api/v1/popular", {{namespace}}::Feeds, :popular
+
+ # Channels
+ get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
+ get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest
+ get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos
+ get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
+ get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
+ get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
+ get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
+ get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses
+ get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
+ get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
+ get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community
+ get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
+ get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search
+
+ # Posts
+ get "/api/v1/post/:id", {{namespace}}::Channels, :post
+ get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments
+
+ # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community
+ get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect
+ get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect
+
+ # Search
+ get "/api/v1/search", {{namespace}}::Search, :search
+ get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions
+ get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag
+
+
+ # Authenticated
+
+ get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
+ post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
+
+ get "/api/v1/auth/export/invidious", {{namespace}}::Authenticated, :export_invidious
+ post "/api/v1/auth/import/invidious", {{namespace}}::Authenticated, :import_invidious
+
+ get "/api/v1/auth/history", {{namespace}}::Authenticated, :get_history
+ post "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_watched
+ delete "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_unwatched
+ delete "/api/v1/auth/history", {{namespace}}::Authenticated, :clear_history
+
+ get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed
+
+ get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions
+ post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel
+ delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel
+
+ get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists
+ post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist
+ patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute
+ delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist
+ post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist
+ delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist
+
+ get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens
+ post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token
+ post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token
+
+ if CONFIG.enable_user_notifications
+ get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ end
+
+ # Misc
+ get "/api/v1/stats", {{namespace}}::Misc, :stats
+ get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist
+ get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist
+ get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes
+ get "/api/v1/resolveurl", {{namespace}}::Misc, :resolve_url
+ {% end %}
+ end
end
diff --git a/src/invidious/search.cr b/src/invidious/search.cr
deleted file mode 100644
index ae106bf6..00000000
--- a/src/invidious/search.cr
+++ /dev/null
@@ -1,254 +0,0 @@
-class ChannelSearchException < InfoException
- getter channel : String
-
- def initialize(@channel)
- end
-end
-
-def channel_search(query, page, channel) : Array(SearchItem)
- response = YT_POOL.client &.get("/channel/#{channel}")
-
- if response.status_code == 404
- response = YT_POOL.client &.get("/user/#{channel}")
- response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404
- initial_data = extract_initial_data(response.body)
- ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
- raise ChannelSearchException.new(channel) if !ucid
- else
- ucid = channel
- end
-
- continuation = produce_channel_search_continuation(ucid, query, page)
- response_json = YoutubeAPI.browse(continuation)
-
- continuation_items = response_json["onResponseReceivedActions"]?
- .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
- return [] of SearchItem if !continuation_items
-
- items = [] of SearchItem
- continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
- extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
- end
-
- return items
-end
-
-def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem)
- return [] of SearchItem if query.empty?
-
- client_config = YoutubeAPI::ClientConfig.new(region: region)
- initial_data = YoutubeAPI.search(query, search_params, client_config: client_config)
-
- return extract_items(initial_data)
-end
-
-def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "",
- duration : String = "", features : Array(String) = [] of String)
- object = {
- "1:varint" => 0_i64,
- "2:embedded" => {} of String => Int64,
- "9:varint" => ((page - 1) * 20).to_i64,
- }
-
- case sort
- when "relevance"
- object["1:varint"] = 0_i64
- when "rating"
- object["1:varint"] = 1_i64
- when "upload_date", "date"
- object["1:varint"] = 2_i64
- when "view_count", "views"
- object["1:varint"] = 3_i64
- else
- raise "No sort #{sort}"
- end
-
- case date
- when "hour"
- object["2:embedded"].as(Hash)["1:varint"] = 1_i64
- when "today"
- object["2:embedded"].as(Hash)["1:varint"] = 2_i64
- when "week"
- object["2:embedded"].as(Hash)["1:varint"] = 3_i64
- when "month"
- object["2:embedded"].as(Hash)["1:varint"] = 4_i64
- when "year"
- object["2:embedded"].as(Hash)["1:varint"] = 5_i64
- else nil # Ignore
- end
-
- case content_type
- when "video"
- object["2:embedded"].as(Hash)["2:varint"] = 1_i64
- when "channel"
- object["2:embedded"].as(Hash)["2:varint"] = 2_i64
- when "playlist"
- object["2:embedded"].as(Hash)["2:varint"] = 3_i64
- when "movie"
- object["2:embedded"].as(Hash)["2:varint"] = 4_i64
- when "show"
- object["2:embedded"].as(Hash)["2:varint"] = 5_i64
- when "all"
- #
- else
- object["2:embedded"].as(Hash)["2:varint"] = 1_i64
- end
-
- case duration
- when "short"
- object["2:embedded"].as(Hash)["3:varint"] = 1_i64
- when "long"
- object["2:embedded"].as(Hash)["3:varint"] = 2_i64
- else nil # Ignore
- end
-
- features.each do |feature|
- case feature
- when "hd"
- object["2:embedded"].as(Hash)["4:varint"] = 1_i64
- when "subtitles"
- object["2:embedded"].as(Hash)["5:varint"] = 1_i64
- when "creative_commons", "cc"
- object["2:embedded"].as(Hash)["6:varint"] = 1_i64
- when "3d"
- object["2:embedded"].as(Hash)["7:varint"] = 1_i64
- when "live", "livestream"
- object["2:embedded"].as(Hash)["8:varint"] = 1_i64
- when "purchased"
- object["2:embedded"].as(Hash)["9:varint"] = 1_i64
- when "4k"
- object["2:embedded"].as(Hash)["14:varint"] = 1_i64
- when "360"
- object["2:embedded"].as(Hash)["15:varint"] = 1_i64
- when "location"
- object["2:embedded"].as(Hash)["23:varint"] = 1_i64
- when "hdr"
- object["2:embedded"].as(Hash)["25:varint"] = 1_i64
- else nil # Ignore
- end
- end
-
- if object["2:embedded"].as(Hash).empty?
- object.delete("2:embedded")
- end
-
- params = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- return params
-end
-
-def produce_channel_search_continuation(ucid, query, page)
- if page <= 1
- idx = 0_i64
- else
- idx = 30_i64 * (page - 1)
- end
-
- object = {
- "80226972:embedded" => {
- "2:string" => ucid,
- "3:base64" => {
- "2:string" => "search",
- "6:varint" => 1_i64,
- "7:varint" => 1_i64,
- "12:varint" => 1_i64,
- "15:base64" => {
- "3:varint" => idx,
- },
- "23:varint" => 0_i64,
- },
- "11:string" => query,
- "35:string" => "browse-feed#{ucid}search",
- },
- }
-
- continuation = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- return continuation
-end
-
-def process_search_query(query, page, user, region)
- if user
- user = user.as(Invidious::User)
- view_name = "subscriptions_#{sha256(user.email)}"
- end
-
- channel = nil
- content_type = "all"
- date = ""
- duration = ""
- features = [] of String
- sort = "relevance"
- subscriptions = nil
-
- operators = query.split(" ").select(&.match(/\w+:[\w,]+/))
- operators.each do |operator|
- key, value = operator.downcase.split(":")
-
- case key
- when "channel", "user"
- channel = operator.split(":")[-1]
- when "content_type", "type"
- content_type = value
- when "date"
- date = value
- when "duration"
- duration = value
- when "feature", "features"
- features = value.split(",")
- when "sort"
- sort = value
- when "subscriptions"
- subscriptions = value == "true"
- else
- operators.delete(operator)
- end
- end
-
- search_query = (query.split(" ") - operators).join(" ")
-
- if channel
- items = channel_search(search_query, page, channel)
- elsif subscriptions
- if view_name
- items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM (
- SELECT *,
- to_tsvector(#{view_name}.title) ||
- to_tsvector(#{view_name}.author)
- as document
- FROM #{view_name}
- ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo)
- else
- items = [] of ChannelVideo
- end
- else
- search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type,
- duration: duration, features: features)
-
- items = search(search_query, search_params, region)
- end
-
- # Light processing to flatten search results out of Categories.
- # They should ideally be supported in the future.
- items_without_category = [] of SearchItem | ChannelVideo
- items.each do |i|
- if i.is_a? Category
- i.contents.each do |nest_i|
- if !nest_i.is_a? Video
- items_without_category << nest_i
- end
- end
- else
- items_without_category << i
- end
- end
-
- {search_query, items_without_category, operators}
-end
diff --git a/src/invidious/search/ctoken.cr b/src/invidious/search/ctoken.cr
new file mode 100644
index 00000000..161065e0
--- /dev/null
+++ b/src/invidious/search/ctoken.cr
@@ -0,0 +1,32 @@
+def produce_channel_search_continuation(ucid, query, page)
+ if page <= 1
+ idx = 0_i64
+ else
+ idx = 30_i64 * (page - 1)
+ end
+
+ object = {
+ "80226972:embedded" => {
+ "2:string" => ucid,
+ "3:base64" => {
+ "2:string" => "search",
+ "6:varint" => 1_i64,
+ "7:varint" => 1_i64,
+ "12:varint" => 1_i64,
+ "15:base64" => {
+ "3:varint" => idx,
+ },
+ "23:varint" => 0_i64,
+ },
+ "11:string" => query,
+ "35:string" => "browse-feed#{ucid}search",
+ },
+ }
+
+ continuation = object.try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+
+ return continuation
+end
diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr
new file mode 100644
index 00000000..bc2715cf
--- /dev/null
+++ b/src/invidious/search/filters.cr
@@ -0,0 +1,376 @@
+require "protodec/utils"
+require "http/params"
+
+module Invidious::Search
+ struct Filters
+ # Values correspond to { "2:embedded": { "1:varint": }}
+ # except for "None" which is only used by us (= nothing selected)
+ enum Date
+ None = 0
+ Hour = 1
+ Today = 2
+ Week = 3
+ Month = 4
+ Year = 5
+ end
+
+ # Values correspond to { "2:embedded": { "2:varint": }}
+ # except for "All" which is only used by us (= nothing selected)
+ enum Type
+ All = 0
+ Video = 1
+ Channel = 2
+ Playlist = 3
+ Movie = 4
+
+ # Has it been removed?
+ # (Not available on youtube's UI)
+ Show = 5
+ end
+
+ # Values correspond to { "2:embedded": { "3:varint": }}
+ # except for "None" which is only used by us (= nothing selected)
+ enum Duration
+ None = 0
+ Short = 1 # "Under 4 minutes"
+ Long = 2 # "Over 20 minutes"
+ Medium = 3 # "4 - 20 minutes"
+ end
+
+ # Note: flag enums automatically generate
+ # "none" and "all" members
+ @[Flags]
+ enum Features
+ Live
+ FourK # "4K"
+ HD
+ Subtitles # "Subtitles/CC"
+ CCommons # "Creative Commons"
+ ThreeSixty # "360°"
+ VR180
+ ThreeD # "3D"
+ HDR
+ Location
+ Purchased
+ end
+
+ # Values correspond to { "1:varint": }
+ enum Sort
+ Relevance = 0
+ Rating = 1
+ Date = 2
+ Views = 3
+ end
+
+ # Parameters are sorted as on Youtube
+ property date : Date
+ property type : Type
+ property duration : Duration
+ property features : Features
+ property sort : Sort
+
+ def initialize(
+ *, # All parameters must be named
+ @date : Date = Date::None,
+ @type : Type = Type::All,
+ @duration : Duration = Duration::None,
+ @features : Features = Features::None,
+ @sort : Sort = Sort::Relevance,
+ )
+ end
+
+ def default? : Bool
+ return @date.none? && @type.all? && @duration.none? && \
+ @features.none? && @sort.relevance?
+ end
+
+ # -------------------
+ # Invidious params
+ # -------------------
+
+ def self.parse_features(raw : Array(String)) : Features
+ # Initialize return variable
+ features = Features.new(0)
+
+ raw.each do |ft|
+ case ft.downcase
+ when "live", "livestream"
+ features = features | Features::Live
+ when "4k" then features = features | Features::FourK
+ when "hd" then features = features | Features::HD
+ when "subtitles" then features = features | Features::Subtitles
+ when "creative_commons", "commons", "cc"
+ features = features | Features::CCommons
+ when "360" then features = features | Features::ThreeSixty
+ when "vr180" then features = features | Features::VR180
+ when "3d" then features = features | Features::ThreeD
+ when "hdr" then features = features | Features::HDR
+ when "location" then features = features | Features::Location
+ when "purchased" then features = features | Features::Purchased
+ end
+ end
+
+ return features
+ end
+
+ def self.format_features(features : Features) : String
+ # Directly return an empty string if there are no features
+ return "" if features.none?
+
+ # Initialize return variable
+ str = [] of String
+
+ str << "live" if features.live?
+ str << "4k" if features.four_k?
+ str << "hd" if features.hd?
+ str << "subtitles" if features.subtitles?
+ str << "commons" if features.c_commons?
+ str << "360" if features.three_sixty?
+ str << "vr180" if features.vr180?
+ str << "3d" if features.three_d?
+ str << "hdr" if features.hdr?
+ str << "location" if features.location?
+ str << "purchased" if features.purchased?
+
+ return str.join(',')
+ end
+
+ def self.from_legacy_filters(str : String) : {Filters, String, String, Bool}
+ # Split search query on spaces
+ members = str.split(' ')
+
+ # Output variables
+ channel = ""
+ filters = Filters.new
+ subscriptions = false
+
+ # Array to hold the non-filter members
+ query = [] of String
+
+ # Parse!
+ members.each do |substr|
+ # Separator operators
+ operators = substr.split(':')
+
+ case operators[0]
+ when "user", "channel"
+ next if operators.size != 2
+ channel = operators[1]
+ #
+ when "type", "content_type"
+ next if operators.size != 2
+ type = Type.parse?(operators[1])
+ filters.type = type if !type.nil?
+ #
+ when "date"
+ next if operators.size != 2
+ date = Date.parse?(operators[1])
+ filters.date = date if !date.nil?
+ #
+ when "duration"
+ next if operators.size != 2
+ duration = Duration.parse?(operators[1])
+ filters.duration = duration if !duration.nil?
+ #
+ when "feature", "features"
+ next if operators.size != 2
+ features = parse_features(operators[1].split(','))
+ filters.features = features if !features.nil?
+ #
+ when "sort"
+ next if operators.size != 2
+ sort = Sort.parse?(operators[1])
+ filters.sort = sort if !sort.nil?
+ #
+ when "subscriptions"
+ next if operators.size != 2
+ subscriptions = {"true", "on", "yes", "1"}.any?(&.== operators[1])
+ #
+ else
+ query << substr
+ end
+ end
+
+ # Re-assemble query (without filters)
+ cleaned_query = query.join(' ')
+
+ return {filters, channel, cleaned_query, subscriptions}
+ end
+
+ def self.from_iv_params(params : HTTP::Params) : Filters
+ # Temporary variables
+ filters = Filters.new
+
+ if type = params["type"]?
+ filters.type = Type.parse?(type) || Type::All
+ params.delete("type")
+ end
+
+ if date = params["date"]?
+ filters.date = Date.parse?(date) || Date::None
+ params.delete("date")
+ end
+
+ if duration = params["duration"]?
+ filters.duration = Duration.parse?(duration) || Duration::None
+ params.delete("duration")
+ end
+
+ features = params.fetch_all("features")
+ if !features.empty?
+ # Un-array input so it can be treated as a comma-separated list
+ features = features[0].split(',') if features.size == 1
+
+ filters.features = parse_features(features) || Features::None
+ params.delete_all("features")
+ end
+
+ if sort = params["sort"]?
+ filters.sort = Sort.parse?(sort) || Sort::Relevance
+ params.delete("sort")
+ end
+
+ return filters
+ end
+
+ def to_iv_params : HTTP::Params
+ # Temporary variables
+ raw_params = {} of String => Array(String)
+
+ raw_params["date"] = [@date.to_s.underscore] if !@date.none?
+ raw_params["type"] = [@type.to_s.underscore] if !@type.all?
+ raw_params["sort"] = [@sort.to_s.underscore] if !@sort.relevance?
+
+ if !@duration.none?
+ raw_params["duration"] = [@duration.to_s.underscore]
+ end
+
+ if !@features.none?
+ raw_params["features"] = [Filters.format_features(@features)]
+ end
+
+ return HTTP::Params.new(raw_params)
+ end
+
+ # -------------------
+ # Youtube params
+ # -------------------
+
+ # Produce the youtube search parameters for the
+ # innertube API (base64-encoded protobuf object).
+ def to_yt_params(page : Int = 1) : String
+ # Initialize the embedded protobuf object
+ embedded = {} of String => Int64
+
+ # Add these field only if associated parameter is selected
+ embedded["1:varint"] = @date.to_i64 if !@date.none?
+ embedded["2:varint"] = @type.to_i64 if !@type.all?
+ embedded["3:varint"] = @duration.to_i64 if !@duration.none?
+
+ if !@features.none?
+ # All features have a value of "1" when enabled, and
+ # the field is omitted when the feature is no selected.
+ embedded["4:varint"] = 1_i64 if @features.includes?(Features::HD)
+ embedded["5:varint"] = 1_i64 if @features.includes?(Features::Subtitles)
+ embedded["6:varint"] = 1_i64 if @features.includes?(Features::CCommons)
+ embedded["7:varint"] = 1_i64 if @features.includes?(Features::ThreeD)
+ embedded["8:varint"] = 1_i64 if @features.includes?(Features::Live)
+ embedded["9:varint"] = 1_i64 if @features.includes?(Features::Purchased)
+ embedded["14:varint"] = 1_i64 if @features.includes?(Features::FourK)
+ embedded["15:varint"] = 1_i64 if @features.includes?(Features::ThreeSixty)
+ embedded["23:varint"] = 1_i64 if @features.includes?(Features::Location)
+ embedded["25:varint"] = 1_i64 if @features.includes?(Features::HDR)
+ embedded["26:varint"] = 1_i64 if @features.includes?(Features::VR180)
+ end
+
+ # Initialize an empty protobuf object
+ object = {} of String => (Int64 | String | Hash(String, Int64))
+
+ # As usual, everything can be omitted if it has no value
+ object["2:embedded"] = embedded if !embedded.empty?
+
+ # Default sort is "relevance", so when this option is selected,
+ # the associated field can be omitted.
+ if !@sort.relevance?
+ object["1:varint"] = @sort.to_i64
+ end
+
+ # Add page number (if provided)
+ if page > 1
+ object["9:varint"] = ((page - 1) * 20).to_i64
+ end
+
+ # Prevent censoring of self harm topics
+ # See https://github.com/iv-org/invidious/issues/4398
+ object["30:varint"] = 1.to_i64
+
+ return object
+ .try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+ end
+
+ # Function to parse the `sp` URL parameter from Youtube
+ # search page. It's a base64-encoded protobuf object.
+ def self.from_yt_params(params : HTTP::Params) : Filters
+ # Initialize output variable
+ filters = Filters.new
+
+ # Get parameter, and check emptyness
+ search_params = params["sp"]?
+
+ if search_params.nil? || search_params.empty?
+ return filters
+ end
+
+ # Decode protobuf object
+ object = search_params
+ .try { |i| URI.decode_www_form(i) }
+ .try { |i| Base64.decode(i) }
+ .try { |i| IO::Memory.new(i) }
+ .try { |i| Protodec::Any.parse(i) }
+
+ # Parse items from embedded object
+ if embedded = object["2:0:embedded"]?
+ # All the following fields (date, type, duration) are optional.
+ if date = embedded["1:0:varint"]?
+ filters.date = Date.from_value?(date.as_i) || Date::None
+ end
+
+ if type = embedded["2:0:varint"]?
+ filters.type = Type.from_value?(type.as_i) || Type::All
+ end
+
+ if duration = embedded["3:0:varint"]?
+ filters.duration = Duration.from_value?(duration.as_i) || Duration::None
+ end
+
+ # All features should have a value of "1" when enabled, and
+ # the field should be omitted when the feature is no selected.
+ features = 0
+ features += (embedded["4:0:varint"]?.try &.as_i == 1_i64) ? Features::HD.value : 0
+ features += (embedded["5:0:varint"]?.try &.as_i == 1_i64) ? Features::Subtitles.value : 0
+ features += (embedded["6:0:varint"]?.try &.as_i == 1_i64) ? Features::CCommons.value : 0
+ features += (embedded["7:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeD.value : 0
+ features += (embedded["8:0:varint"]?.try &.as_i == 1_i64) ? Features::Live.value : 0
+ features += (embedded["9:0:varint"]?.try &.as_i == 1_i64) ? Features::Purchased.value : 0
+ features += (embedded["14:0:varint"]?.try &.as_i == 1_i64) ? Features::FourK.value : 0
+ features += (embedded["15:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeSixty.value : 0
+ features += (embedded["23:0:varint"]?.try &.as_i == 1_i64) ? Features::Location.value : 0
+ features += (embedded["25:0:varint"]?.try &.as_i == 1_i64) ? Features::HDR.value : 0
+ features += (embedded["26:0:varint"]?.try &.as_i == 1_i64) ? Features::VR180.value : 0
+
+ filters.features = Features.from_value?(features) || Features::None
+ end
+
+ if sort = object["1:0:varint"]?
+ filters.sort = Sort.from_value?(sort.as_i) || Sort::Relevance
+ end
+
+ # Remove URL parameter and return result
+ params.delete("sp")
+ return filters
+ end
+ end
+end
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
new file mode 100644
index 00000000..25edb936
--- /dev/null
+++ b/src/invidious/search/processors.cr
@@ -0,0 +1,56 @@
+module Invidious::Search
+ module Processors
+ extend self
+
+ # Regular search (`/search` endpoint)
+ def regular(query : Query) : Array(SearchItem)
+ search_params = query.filters.to_yt_params(page: query.page)
+
+ client_config = YoutubeAPI::ClientConfig.new(region: query.region)
+ initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
+
+ items, _ = extract_items(initial_data)
+ return items.reject!(Category)
+ end
+
+ # Search a youtube channel
+ # TODO: clean code, and rely more on YoutubeAPI
+ def channel(query : Query) : Array(SearchItem)
+ response = YT_POOL.client &.get("/channel/#{query.channel}")
+
+ if response.status_code == 404
+ response = YT_POOL.client &.get("/user/#{query.channel}")
+ response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
+ initial_data = extract_initial_data(response.body)
+ ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
+ raise ChannelSearchException.new(query.channel) if !ucid
+ else
+ ucid = query.channel
+ end
+
+ continuation = produce_channel_search_continuation(ucid, query.text, query.page)
+ response_json = YoutubeAPI.browse(continuation)
+
+ items, _ = extract_items(response_json, "", ucid)
+ return items.reject!(Category)
+ end
+
+ # Search inside of user subscriptions
+ def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
+ view_name = "subscriptions_#{sha256(user.email)}"
+
+ return PG_DB.query_all("
+ SELECT id,title,published,updated,ucid,author,length_seconds
+ FROM (
+ SELECT *,
+ to_tsvector(#{view_name}.title) ||
+ to_tsvector(#{view_name}.author)
+ as document
+ FROM #{view_name}
+ ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
+ query.text, (query.page - 1) * 20,
+ as: ChannelVideo
+ )
+ end
+ end
+end
diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr
new file mode 100644
index 00000000..94a92e23
--- /dev/null
+++ b/src/invidious/search/query.cr
@@ -0,0 +1,168 @@
+module Invidious::Search
+ class Query
+ enum Type
+ # Types related to YouTube
+ Regular # Youtube search page
+ Channel # Youtube channel search box
+
+ # Types specific to Invidious
+ Subscriptions # Search user subscriptions
+ Playlist # "Add playlist item" search
+ end
+
+ getter type : Type = Type::Regular
+
+ @raw_query : String
+ @query : String = ""
+
+ property filters : Filters = Filters.new
+ property page : Int32
+ property region : String?
+ property channel : String = ""
+
+ # Flag that indicates if the smart search features have been disabled.
+ @inhibit_ssf : Bool = false
+
+ # Return true if @raw_query is either `nil` or empty
+ private def empty_raw_query?
+ return @raw_query.empty?
+ end
+
+ # Same as `empty_raw_query?`, but named for external use
+ def empty?
+ return self.empty_raw_query?
+ end
+
+ # Getter for the query string.
+ # It is named `text` to reduce confusion (`search_query.text` makes more
+ # sense than `search_query.query`)
+ def text
+ return @query
+ end
+
+ # Initialize a new search query.
+ # Parameters are used to get the query string, the page number
+ # and the search filters (if any). Type tells this function
+ # where it is being called from (See `Type` above).
+ def initialize(
+ params : HTTP::Params,
+ @type : Type = Type::Regular,
+ @region : String? = nil,
+ )
+ # Get the raw search query string (common to all search types). In
+ # Regular search mode, also look for the `search_query` URL parameter
+ _raw_query = params["q"]?
+ _raw_query ||= params["search_query"]? if @type.regular?
+ _raw_query ||= ""
+
+ # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs.
+ @raw_query = _raw_query.strip
+
+ # Check for smart features (ex: URL search) inhibitor (backslash).
+ # If inhibitor is present, remove it.
+ if @raw_query.starts_with?('\\')
+ @inhibit_ssf = true
+ @raw_query = @raw_query[1..]
+ end
+
+ # Get the page number (also common to all search types)
+ @page = params["page"]?.try &.to_i? || 1
+
+ # Stop here if raw query is empty
+ # NOTE: maybe raise in the future?
+ return if self.empty_raw_query?
+
+ # Specific handling
+ case @type
+ when .channel?
+ # In "channel search" mode, filters are ignored, but we still parse
+ # the query prevent transmission of legacy filters to youtube.
+ #
+ _, _, @query, _ = Filters.from_legacy_filters(@raw_query)
+ #
+ when .playlist?
+ # In "add playlist item" mode, filters are parsed from the query
+ # string itself (legacy), and the channel is ignored.
+ #
+ @filters, _, @query, _ = Filters.from_legacy_filters(@raw_query)
+ #
+ when .subscriptions?, .regular?
+ if params["sp"]?
+ # Parse the `sp` URL parameter (youtube compatibility)
+ @filters = Filters.from_yt_params(params)
+ @query = @raw_query || ""
+ else
+ # Parse invidious URL parameters (sort, date, etc...)
+ @filters = Filters.from_iv_params(params)
+ @channel = params["channel"]? || ""
+
+ if @filters.default? && @raw_query.index(/\w:\w/)
+ # Parse legacy filters from query
+ @filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query)
+ else
+ @query = @raw_query || ""
+ end
+
+ if !@channel.empty?
+ # Switch to channel search mode (filters will be ignored)
+ @type = Type::Channel
+ elsif subs
+ # Switch to subscriptions search mode
+ @type = Type::Subscriptions
+ end
+ end
+ end
+ end
+
+ # Run the search query using the corresponding search processor.
+ # Returns either the results or an empty array of `SearchItem`.
+ def process(user : Invidious::User? = nil) : Array(SearchItem) | Array(ChannelVideo)
+ items = [] of SearchItem
+
+ # Don't bother going further if search query is empty
+ return items if self.empty_raw_query?
+
+ case @type
+ when .regular?, .playlist?
+ items = Processors.regular(self)
+ #
+ when .channel?
+ items = Processors.channel(self)
+ #
+ when .subscriptions?
+ if user
+ items = Processors.subscriptions(self, user.as(Invidious::User))
+ end
+ end
+
+ return items
+ end
+
+ # Return the HTTP::Params corresponding to this Query (invidious format)
+ def to_http_params : HTTP::Params
+ params = @filters.to_iv_params
+
+ params["q"] = @query
+ params["channel"] = @channel if !@channel.empty?
+
+ return params
+ end
+
+ # Checks if the query is a standalone URL
+ def url? : Bool
+ # If the smart features have been inhibited, don't go further.
+ return false if @inhibit_ssf
+
+ # Only supported in regular search mode
+ return false if !@type.regular?
+
+ # If filters are present, that's a regular search
+ return false if !@filters.default?
+
+ # Simple heuristics: domain name
+ return @raw_query.starts_with?(
+ /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\//
+ )
+ end
+ end
+end
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index 1f957081..d14cde5d 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -4,11 +4,12 @@ def fetch_trending(trending_type, region, locale)
plid = nil
- if trending_type == "Music"
+ case trending_type.try &.downcase
+ when "music"
params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D"
- elsif trending_type == "Gaming"
+ when "gaming"
params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D"
- elsif trending_type == "Movies"
+ when "movies"
params = "4gIKGgh0cmFpbGVycw%3D%3D"
else # Default
params = ""
@@ -16,7 +17,26 @@ def fetch_trending(trending_type, region, locale)
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.browse("FEtrending", params: params, client_config: client_config)
- trending = extract_videos(initial_data)
- return {trending, plid}
+ items, _ = extract_items(initial_data)
+
+ extracted = [] of SearchItem
+
+ deduplicate = items.size > 1
+
+ items.each do |itm|
+ if itm.is_a?(Category)
+ # Ignore the smaller categories, as they generally contain a sponsored
+ # channel, which brings a lot of noise on the trending page.
+ # See: https://github.com/iv-org/invidious/issues/2989
+ next if (itm.contents.size < 24 && deduplicate)
+
+ extracted.concat itm.contents.select(SearchItem)
+ else
+ extracted << itm
+ end
+ end
+
+ # Deduplicate items before returning results
+ return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
end
diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr
index 8a0f67e5..b175c3b9 100644
--- a/src/invidious/user/captcha.cr
+++ b/src/invidious/user/captcha.cr
@@ -4,8 +4,6 @@ struct Invidious::User
module Captcha
extend self
- private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com")
-
def generate_image(key)
second = Random::Secure.rand(12)
second_angle = second * 30
@@ -60,19 +58,5 @@ struct Invidious::User
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)},
}
end
-
- def generate_text(key)
- response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body)
- response = JSON.parse(response)
-
- tokens = response["a"].as_a.map do |answer|
- generate_response(answer.as_s, {":login"}, key, use_nonce: true)
- end
-
- return {
- question: response["q"].as_s,
- tokens: tokens,
- }
- end
end
end
diff --git a/src/invidious/user/cookies.cr b/src/invidious/user/cookies.cr
index 99df1b07..654efc15 100644
--- a/src/invidious/user/cookies.cr
+++ b/src/invidious/user/cookies.cr
@@ -17,7 +17,8 @@ struct Invidious::User
value: sid,
expires: Time.utc + 2.years,
secure: SECURE,
- http_only: true
+ http_only: true,
+ samesite: HTTP::Cookie::SameSite::Lax
)
end
@@ -30,7 +31,8 @@ struct Invidious::User
value: URI.encode_www_form(preferences.to_json),
expires: Time.utc + 2.years,
secure: SECURE,
- http_only: true
+ http_only: false,
+ samesite: HTTP::Cookie::SameSite::Lax
)
end
end
diff --git a/src/invidious/user/exports.cr b/src/invidious/user/exports.cr
new file mode 100644
index 00000000..b52503c9
--- /dev/null
+++ b/src/invidious/user/exports.cr
@@ -0,0 +1,35 @@
+struct Invidious::User
+ module Export
+ extend self
+
+ def to_invidious(user : User)
+ playlists = Invidious::Database::Playlists.select_like_iv(user.email)
+
+ return JSON.build do |json|
+ json.object do
+ json.field "subscriptions", user.subscriptions
+ json.field "watch_history", user.watched
+ json.field "preferences", user.preferences
+ json.field "playlists" do
+ json.array do
+ playlists.each do |playlist|
+ json.object do
+ json.field "title", playlist.title
+ json.field "description", html_to_content(playlist.description_html)
+ json.field "privacy", playlist.privacy.to_s
+ json.field "videos" do
+ json.array do
+ Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: CONFIG.playlist_length_limit).each do |video_id|
+ json.string video_id
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end # module
+end
diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr
index f8b9e4e4..007eb666 100644
--- a/src/invidious/user/imports.cr
+++ b/src/invidious/user/imports.cr
@@ -6,7 +6,7 @@ struct Invidious::User
# Parse a youtube CSV subscription file
def parse_subscription_export_csv(csv_content : String)
- rows = CSV.new(csv_content, headers: true)
+ rows = CSV.new(csv_content.strip('\n'), headers: true)
subscriptions = Array(String).new
# Counter to limit the amount of imports.
@@ -30,6 +30,60 @@ struct Invidious::User
return subscriptions
end
+ def parse_playlist_export_csv(user : User, raw_input : String)
+ # Split the input into head and body content
+ raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true)
+
+ # Create the playlist from the head content
+ csv_head = CSV.new(raw_head.strip('\n'), headers: true)
+ csv_head.next
+ title = csv_head[4]
+ description = csv_head[5]
+ visibility = csv_head[6]
+
+ if visibility.compare("Public", case_insensitive: true) == 0
+ privacy = PlaylistPrivacy::Public
+ else
+ privacy = PlaylistPrivacy::Private
+ end
+
+ playlist = create_playlist(title, privacy, user)
+ Invidious::Database::Playlists.update_description(playlist.id, description)
+
+ # Add each video to the playlist from the body content
+ csv_body = CSV.new(raw_body.strip('\n'), headers: true)
+ csv_body.each do |row|
+ video_id = row[0]
+ if playlist
+ next if !video_id
+ next if video_id == "Video Id"
+
+ begin
+ video = get_video(video_id)
+ rescue ex
+ next
+ end
+
+ playlist_video = PlaylistVideo.new({
+ title: video.title,
+ id: video.id,
+ author: video.author,
+ ucid: video.ucid,
+ length_seconds: video.length_seconds,
+ published: video.published,
+ plid: playlist.id,
+ live_now: video.live_now,
+ index: Random::Secure.rand(0_i64..Int64::MAX),
+ })
+
+ Invidious::Database::PlaylistVideos.insert(playlist_video)
+ Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
+ end
+ end
+
+ return playlist
+ end
+
# -------------------
# Invidious
# -------------------
@@ -48,7 +102,7 @@ struct Invidious::User
if data["watch_history"]?
user.watched += data["watch_history"].as_a.map(&.as_s)
- user.watched.uniq!
+ user.watched.reverse!.uniq!.reverse!
Invidious::Database::Users.update_watch_history(user)
end
@@ -61,7 +115,7 @@ struct Invidious::User
playlists.each do |item|
title = item["title"]?.try &.as_s?.try &.delete("<>")
description = item["description"]?.try &.as_s?.try &.delete("\r")
- privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
+ privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state }
next if !title
next if !description
@@ -70,14 +124,16 @@ struct Invidious::User
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
- videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
- raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
+ item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
+ if idx > CONFIG.playlist_length_limit
+ raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
+ end
video_id = video_id.try &.as_s?
next if !video_id
begin
- video = get_video(video_id)
+ video = get_video(video_id, false)
rescue ex
next
end
@@ -105,7 +161,7 @@ struct Invidious::User
# Youtube
# -------------------
- private def is_opml?(mimetype : String, extension : String)
+ private def opml?(mimetype : String, extension : String)
opml_mimetypes = [
"application/xml",
"text/xml",
@@ -123,10 +179,10 @@ struct Invidious::User
def from_youtube(user : User, body : String, filename : String, type : String) : Bool
extension = filename.split(".").last
- if is_opml?(type, extension)
+ if opml?(type, extension)
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
- channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
+ channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0]
end
elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body)
@@ -147,6 +203,41 @@ struct Invidious::User
return true
end
+ def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool
+ extension = filename.split(".").last
+
+ if extension == "csv" || type == "text/csv"
+ playlist = parse_playlist_export_csv(user, body)
+ if playlist
+ return true
+ else
+ return false
+ end
+ else
+ return false
+ end
+ end
+
+ def from_youtube_wh(user : User, body : String, filename : String, type : String) : Bool
+ extension = filename.split(".").last
+
+ if extension == "json" || type == "application/json"
+ data = JSON.parse(body)
+ watched = data.as_a.compact_map do |item|
+ next unless url = item["titleUrl"]?
+ next unless match = url.as_s.match(/\?v=(?[a-zA-Z0-9_-]+)$/)
+ match["video_id"]
+ end
+ watched.reverse! # YouTube have newest first
+ user.watched += watched
+ user.watched.uniq!
+ Invidious::Database::Users.update_watch_history(user)
+ return true
+ else
+ return false
+ end
+ end
+
# -------------------
# Freetube
# -------------------
@@ -157,8 +248,12 @@ struct Invidious::User
subs = matches.map(&.["channel_id"])
if subs.empty?
- data = JSON.parse(body)["subscriptions"]
- subs = data.as_a.map(&.["id"].as_s)
+ profiles = body.split('\n', remove_empty: true)
+ profiles.each do |profile|
+ if data = JSON.parse(profile)["subscriptions"]?
+ subs += data.as_a.map(&.["id"].as_s)
+ end
+ end
end
user.subscriptions += subs
@@ -195,42 +290,39 @@ struct Invidious::User
end
def from_newpipe(user : User, body : String) : Bool
- io = IO::Memory.new(body)
+ Compress::Zip::File.open(IO::Memory.new(body), true) do |file|
+ entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" }
+ return false if entry.nil?
+ entry.open do |file_io|
+ # Ensure max size of 4MB
+ io_sized = IO::Sized.new(file_io, 0x400000)
- Compress::Zip::File.open(io) do |file|
- file.entries.each do |entry|
- entry.open do |file_io|
- # Ensure max size of 4MB
- io_sized = IO::Sized.new(file_io, 0x400000)
+ begin
+ temp = File.tempfile(".db") do |tempfile|
+ begin
+ File.write(tempfile.path, io_sized.gets_to_end)
+ rescue
+ return false
+ end
- next if entry.filename != "newpipe.db"
+ DB.open("sqlite3://" + tempfile.path) do |db|
+ user.watched += db.query_all("SELECT url FROM streams", as: String)
+ .map(&.lchop("https://www.youtube.com/watch?v="))
- tempfile = File.tempfile(".db")
+ user.watched.uniq!
+ Invidious::Database::Users.update_watch_history(user)
- begin
- File.write(tempfile.path, io_sized.gets_to_end)
- rescue
- return false
+ user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
+ .map(&.lchop("https://www.youtube.com/channel/"))
+
+ user.subscriptions.uniq!
+ user.subscriptions = get_batch_channels(user.subscriptions)
+
+ Invidious::Database::Users.update_subscriptions(user)
+ end
end
-
- db = DB.open("sqlite3://" + tempfile.path)
-
- user.watched += db.query_all("SELECT url FROM streams", as: String)
- .map(&.lchop("https://www.youtube.com/watch?v="))
-
- user.watched.uniq!
- Invidious::Database::Users.update_watch_history(user)
-
- user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
- .map(&.lchop("https://www.youtube.com/channel/"))
-
- user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions)
-
- Invidious::Database::Users.update_subscriptions(user)
-
- db.close
- tempfile.delete
+ ensure
+ temp.delete if !temp.nil?
end
end
end
diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr
index b3059403..0a8525f3 100644
--- a/src/invidious/user/preferences.cr
+++ b/src/invidious/user/preferences.cr
@@ -4,6 +4,7 @@ struct Preferences
property annotations : Bool = CONFIG.default_user_preferences.annotations
property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
+ property preload : Bool = CONFIG.default_user_preferences.preload
property autoplay : Bool = CONFIG.default_user_preferences.autoplay
property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index b763596b..65566d20 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -3,75 +3,6 @@ require "crypto/bcrypt/password"
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
-def get_user(sid, headers, refresh = true)
- if email = Invidious::Database::SessionIDs.select_email(sid)
- user = Invidious::Database::Users.select!(email: email)
-
- if refresh && Time.utc - user.updated > 1.minute
- user, sid = fetch_user(sid, headers)
-
- Invidious::Database::Users.insert(user, update_on_conflict: true)
- Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
-
- begin
- view_name = "subscriptions_#{sha256(user.email)}"
- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
- rescue ex
- end
- end
- else
- user, sid = fetch_user(sid, headers)
-
- Invidious::Database::Users.insert(user, update_on_conflict: true)
- Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
-
- begin
- view_name = "subscriptions_#{sha256(user.email)}"
- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
- rescue ex
- end
- end
-
- return user, sid
-end
-
-def fetch_user(sid, headers)
- feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
- feed = XML.parse_html(feed.body)
-
- channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
- if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
- nil
- else
- channel["href"].lstrip("/channel/")
- end
- end
-
- channels = get_batch_channels(channels)
-
- email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
- if email
- email = email.content.strip
- else
- email = ""
- end
-
- token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
-
- user = Invidious::User.new({
- updated: Time.utc,
- notifications: [] of String,
- subscriptions: channels,
- email: email,
- preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
- password: nil,
- token: token,
- watched: [] of String,
- feed_needs_update: true,
- })
- return user, sid
-end
-
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
@@ -91,38 +22,6 @@ def create_user(sid, email, password)
return user, sid
end
-def subscribe_ajax(channel_id, action, env_headers)
- headers = HTTP::Headers.new
- headers["Cookie"] = env_headers["Cookie"]
-
- html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
-
- cookies = HTTP::Cookies.from_client_headers(headers)
- html.cookies.each do |cookie|
- if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
- if cookies[cookie.name]?
- cookies[cookie.name] = cookie
- else
- cookies << cookie
- end
- end
- end
- headers = cookies.add_request_headers(headers)
-
- if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
- session_token = match["session_token"]
-
- headers["content-type"] = "application/x-www-form-urlencoded"
-
- post_req = {
- session_token: session_token,
- }
- post_url = "/subscription_ajax?#{action}=1&c=#{channel_id}"
-
- YT_POOL.client &.post(post_url, headers, form: post_req)
- end
-end
-
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 81fce5b8..348a0a66 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -1,280 +1,22 @@
-CAPTION_LANGUAGES = {
- "",
- "English",
- "English (auto-generated)",
- "English (United Kingdom)",
- "English (United States)",
- "Afrikaans",
- "Albanian",
- "Amharic",
- "Arabic",
- "Armenian",
- "Azerbaijani",
- "Bangla",
- "Basque",
- "Belarusian",
- "Bosnian",
- "Bulgarian",
- "Burmese",
- "Cantonese (Hong Kong)",
- "Catalan",
- "Cebuano",
- "Chinese",
- "Chinese (China)",
- "Chinese (Hong Kong)",
- "Chinese (Simplified)",
- "Chinese (Taiwan)",
- "Chinese (Traditional)",
- "Corsican",
- "Croatian",
- "Czech",
- "Danish",
- "Dutch",
- "Dutch (auto-generated)",
- "Esperanto",
- "Estonian",
- "Filipino",
- "Finnish",
- "French",
- "French (auto-generated)",
- "Galician",
- "Georgian",
- "German",
- "German (auto-generated)",
- "Greek",
- "Gujarati",
- "Haitian Creole",
- "Hausa",
- "Hawaiian",
- "Hebrew",
- "Hindi",
- "Hmong",
- "Hungarian",
- "Icelandic",
- "Igbo",
- "Indonesian",
- "Indonesian (auto-generated)",
- "Interlingue",
- "Irish",
- "Italian",
- "Italian (auto-generated)",
- "Japanese",
- "Japanese (auto-generated)",
- "Javanese",
- "Kannada",
- "Kazakh",
- "Khmer",
- "Korean",
- "Korean (auto-generated)",
- "Kurdish",
- "Kyrgyz",
- "Lao",
- "Latin",
- "Latvian",
- "Lithuanian",
- "Luxembourgish",
- "Macedonian",
- "Malagasy",
- "Malay",
- "Malayalam",
- "Maltese",
- "Maori",
- "Marathi",
- "Mongolian",
- "Nepali",
- "Norwegian Bokmål",
- "Nyanja",
- "Pashto",
- "Persian",
- "Polish",
- "Portuguese",
- "Portuguese (auto-generated)",
- "Portuguese (Brazil)",
- "Punjabi",
- "Romanian",
- "Russian",
- "Russian (auto-generated)",
- "Samoan",
- "Scottish Gaelic",
- "Serbian",
- "Shona",
- "Sindhi",
- "Sinhala",
- "Slovak",
- "Slovenian",
- "Somali",
- "Southern Sotho",
- "Spanish",
- "Spanish (auto-generated)",
- "Spanish (Latin America)",
- "Spanish (Mexico)",
- "Spanish (Spain)",
- "Sundanese",
- "Swahili",
- "Swedish",
- "Tajik",
- "Tamil",
- "Telugu",
- "Thai",
- "Turkish",
- "Turkish (auto-generated)",
- "Ukrainian",
- "Urdu",
- "Uzbek",
- "Vietnamese",
- "Vietnamese (auto-generated)",
- "Welsh",
- "Western Frisian",
- "Xhosa",
- "Yiddish",
- "Yoruba",
- "Zulu",
-}
-
-REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"}
-
-# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
-VIDEO_FORMATS = {
- "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
- "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
- "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
- "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
- "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
- "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
-
- "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
- "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
-
- # 3D videos
- "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
-
- # Apple HTTP Live Streaming
- "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
- "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
- "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
-
- # DASH mp4 video
- "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
- "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
- "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
- "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
- "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
- "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
- "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
- "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
- "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
- "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
- "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
- "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
-
- # Dash mp4 audio
- "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
- "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
- "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
- "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
- "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
- "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
- "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
-
- # Dash webm
- "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
- "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
- "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
- "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
- "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
- "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
- # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
- "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
- "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
- "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
-
- # Dash webm audio
- "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
- "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
-
- # Dash webm audio with opus inside
- "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
- "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
- "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
-
- # av01 video only formats sometimes served with "unknown" codecs
- "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
- "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
- "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
- "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
-}
-
-struct VideoPreferences
- include JSON::Serializable
-
- property annotations : Bool
- property autoplay : Bool
- property comments : Array(String)
- property continue : Bool
- property continue_autoplay : Bool
- property controls : Bool
- property listen : Bool
- property local : Bool
- property preferred_captions : Array(String)
- property player_style : String
- property quality : String
- property quality_dash : String
- property raw : Bool
- property region : String?
- property related_videos : Bool
- property speed : Float32 | Float64
- property video_end : Float64 | Int32
- property video_loop : Bool
- property extend_desc : Bool
- property video_start : Float64 | Int32
- property volume : Int32
- property vr_mode : Bool
- property save_player_pos : Bool
+enum VideoType
+ Video
+ Livestream
+ Scheduled
end
struct Video
include DB::Serializable
+ # Version of the JSON structure
+ # It prevents us from loading an incompatible version from cache
+ # (either newer or older, if instances with different versions run
+ # concurrently, e.g during a version upgrade rollout).
+ #
+ # NOTE: don't forget to bump this number if any change is made to
+ # the `params` structure in videos/parser.cr!!!
+ #
+ SCHEMA_VERSION = 3
+
property id : String
@[DB::Field(converter: Video::JSONConverter)]
@@ -282,13 +24,7 @@ struct Video
property updated : Time
@[DB::Field(ignore: true)]
- property captions : Array(Caption)?
-
- @[DB::Field(ignore: true)]
- property adaptive_fmts : Array(Hash(String, JSON::Any))?
-
- @[DB::Field(ignore: true)]
- property fmt_stream : Array(Hash(String, JSON::Any))?
+ @captions = [] of Invidious::Videos::Captions::Metadata
@[DB::Field(ignore: true)]
property description : String?
@@ -299,274 +35,49 @@ struct Video
end
end
+ # Methods for API v1 JSON
+
def to_json(locale : String?, json : JSON::Builder)
- json.object do
- json.field "type", "video"
-
- json.field "title", self.title
- json.field "videoId", self.id
-
- json.field "error", info["reason"] if info["reason"]?
-
- json.field "videoThumbnails" do
- generate_thumbnails(json, self.id)
- end
- json.field "storyboards" do
- generate_storyboards(json, self.id, self.storyboards)
- end
-
- json.field "description", self.description
- json.field "descriptionHtml", self.description_html
- json.field "published", self.published.to_unix
- json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
- json.field "keywords", self.keywords
-
- json.field "viewCount", self.views
- json.field "likeCount", self.likes
- json.field "dislikeCount", self.dislikes
-
- json.field "paid", self.paid
- json.field "premium", self.premium
- json.field "isFamilyFriendly", self.is_family_friendly
- json.field "allowedRegions", self.allowed_regions
- json.field "genre", self.genre
- json.field "genreUrl", self.genre_url
-
- json.field "author", self.author
- json.field "authorId", self.ucid
- json.field "authorUrl", "/channel/#{self.ucid}"
-
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
-
- json.field "subCountText", self.sub_count_text
-
- json.field "lengthSeconds", self.length_seconds
- json.field "allowRatings", self.allow_ratings
- json.field "rating", self.average_rating
- json.field "isListed", self.is_listed
- json.field "liveNow", self.live_now
- json.field "isUpcoming", self.is_upcoming
-
- if self.premiere_timestamp
- json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
- end
-
- if hlsvp = self.hls_manifest_url
- hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
- json.field "hlsUrl", hlsvp
- end
-
- json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
-
- json.field "adaptiveFormats" do
- json.array do
- self.adaptive_fmts.each do |fmt|
- json.object do
- json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}"
- json.field "bitrate", fmt["bitrate"].as_i.to_s
- json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}"
- json.field "url", fmt["url"]
- json.field "itag", fmt["itag"].as_i.to_s
- json.field "type", fmt["mimeType"]
- json.field "clen", fmt["contentLength"]
- json.field "lmt", fmt["lastModified"]
- json.field "projectionType", fmt["projectionType"]
-
- fmt_info = itag_to_metadata?(fmt["itag"])
- if fmt_info
- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
- json.field "fps", fps
- json.field "container", fmt_info["ext"]
- json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
- if fmt_info["height"]?
- json.field "resolution", "#{fmt_info["height"]}p"
-
- quality_label = "#{fmt_info["height"]}p"
- if fps > 30
- quality_label += "60"
- end
- json.field "qualityLabel", quality_label
-
- if fmt_info["width"]?
- json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
- end
- end
- end
- end
- end
- end
- end
-
- json.field "formatStreams" do
- json.array do
- self.fmt_stream.each do |fmt|
- json.object do
- json.field "url", fmt["url"]
- json.field "itag", fmt["itag"].as_i.to_s
- json.field "type", fmt["mimeType"]
- json.field "quality", fmt["quality"]
-
- fmt_info = itag_to_metadata?(fmt["itag"])
- if fmt_info
- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
- json.field "fps", fps
- json.field "container", fmt_info["ext"]
- json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
- if fmt_info["height"]?
- json.field "resolution", "#{fmt_info["height"]}p"
-
- quality_label = "#{fmt_info["height"]}p"
- if fps > 30
- quality_label += "60"
- end
- json.field "qualityLabel", quality_label
-
- if fmt_info["width"]?
- json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
- end
- end
- end
- end
- end
- end
- end
-
- json.field "captions" do
- json.array do
- self.captions.each do |caption|
- json.object do
- json.field "label", caption.name
- json.field "language_code", caption.language_code
- json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}"
- end
- end
- end
- end
-
- json.field "recommendedVideos" do
- json.array do
- self.related_videos.each do |rv|
- if rv["id"]?
- json.object do
- json.field "videoId", rv["id"]
- json.field "title", rv["title"]
- json.field "videoThumbnails" do
- generate_thumbnails(json, rv["id"])
- end
-
- json.field "author", rv["author"]
- json.field "authorUrl", "/channel/#{rv["ucid"]?}"
- json.field "authorId", rv["ucid"]?
- if rv["author_thumbnail"]?
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
- end
-
- json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
- json.field "viewCountText", rv["short_view_count"]?
- json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
- end
- end
- end
- end
- end
- end
+ Invidious::JSONify::APIv1.video(self, json, locale: locale)
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
- JSON.build { |json| to_json(locale, json) }
+ JSON.build do |json|
+ Invidious::JSONify::APIv1.video(self, json, locale: locale)
+ end
end
def to_json(json : JSON::Builder | Nil = nil)
to_json(nil, json)
end
- def title
- info["videoDetails"]["title"]?.try &.as_s || ""
+ # Misc methods
+
+ def video_type : VideoType
+ video_type = info["videoType"]?.try &.as_s || "video"
+ return VideoType.parse?(video_type) || VideoType::Video
end
- def ucid
- info["videoDetails"]["channelId"]?.try &.as_s || ""
- end
-
- def author
- info["videoDetails"]["author"]?.try &.as_s || ""
- end
-
- def length_seconds : Int32
- info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i ||
- info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
- end
-
- def views : Int64
- info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
- end
-
- def likes : Int64
- info["likes"]?.try &.as_i64 || 0_i64
- end
-
- def dislikes : Int64
- info["dislikes"]?.try &.as_i64 || 0_i64
- end
-
- def average_rating : Float64
- # (likes / (likes + dislikes) * 4 + 1)
- info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0
+ def schema_version : Int
+ return info["version"]?.try &.as_i || 1
end
def published : Time
- info
- .dig?("microformat", "playerMicroformatRenderer", "publishDate")
+ return info["published"]?
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
end
def published=(other : Time)
- info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
- end
-
- def allow_ratings
- r = info["videoDetails"]["allowRatings"]?.try &.as_bool
- r.nil? ? false : r
+ info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
end
def live_now
- info["microformat"]?.try &.["playerMicroformatRenderer"]?
- .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false
+ return (self.video_type == VideoType::Livestream)
end
- def is_listed
- info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
- end
-
- def is_upcoming
- info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
+ def post_live_dvr
+ return info["isPostLiveDvr"].as_bool
end
def premiere_timestamp : Time?
@@ -575,68 +86,30 @@ struct Video
.try { |t| Time.parse_rfc3339(t.as_s) }
end
- def keywords
- info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
- end
-
def related_videos
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
end
- def allowed_regions
- info
- .dig("microformat", "playerMicroformatRenderer", "availableCountries")
- .try &.as_a.map &.as_s || [] of String
- end
+ # Methods for parsing streaming data
- def author_thumbnail : String
- info["authorThumbnail"]?.try &.as_s || ""
- end
-
- def sub_count_text : String
- info["subCountText"]?.try &.as_s || "-"
- end
-
- def fmt_stream
- return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
-
- fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
- fmt_stream.each do |fmt|
- if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
- s.each do |k, v|
- fmt[k] = JSON::Any.new(v)
- end
- fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
- end
-
- fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
- fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
+ def fmt_stream : Array(Hash(String, JSON::Any))
+ if formats = info.dig?("streamingData", "formats")
+ return formats
+ .as_a.map(&.as_h)
+ .sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ else
+ return [] of Hash(String, JSON::Any)
end
- fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
- @fmt_stream = fmt_stream
- return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
- def adaptive_fmts
- return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
- fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
- fmt_stream.each do |fmt|
- if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
- s.each do |k, v|
- fmt[k] = JSON::Any.new(v)
- end
- fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
- end
-
- fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
- fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
+ def adaptive_fmts : Array(Hash(String, JSON::Any))
+ if formats = info.dig?("streamingData", "adaptiveFormats")
+ return formats
+ .as_a.map(&.as_h)
+ .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 }
+ else
+ return [] of Hash(String, JSON::Any)
end
- # See https://github.com/TeamNewPipe/NewPipe/issues/2415
- # Some streams are segmented by URL `sq/` rather than index, for now we just filter them out
- fmt_stream.reject! { |f| !f["indexRange"]? }
- fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
- @adaptive_fmts = fmt_stream
- return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
@@ -647,429 +120,178 @@ struct Video
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
+ # Misc. methods
+
def storyboards
- storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
- .try &.as_s.split("|")
-
- if !storyboards
- if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
- return [{
- url: storyboard.split("#")[0],
- width: 106,
- height: 60,
- count: -1,
- interval: 5000,
- storyboard_width: 3,
- storyboard_height: 3,
- storyboard_count: -1,
- }]
- end
- end
-
- items = [] of NamedTuple(
- url: String,
- width: Int32,
- height: Int32,
- count: Int32,
- interval: Int32,
- storyboard_width: Int32,
- storyboard_height: Int32,
- storyboard_count: Int32)
-
- return items if !storyboards
-
- url = URI.parse(storyboards.shift)
- params = HTTP::Params.parse(url.query || "")
-
- storyboards.each_with_index do |sb, i|
- width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
- params["sigh"] = sigh
- url.query = params.to_s
-
- width = width.to_i
- height = height.to_i
- count = count.to_i
- interval = interval.to_i
- storyboard_width = storyboard_width.to_i
- storyboard_height = storyboard_height.to_i
- storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
-
- items << {
- url: url.to_s.sub("$L", i).sub("$N", "M$M"),
- width: width,
- height: height,
- count: count,
- interval: interval,
- storyboard_width: storyboard_width,
- storyboard_height: storyboard_height,
- storyboard_count: storyboard_count,
- }
- end
-
- items
+ container = info.dig?("storyboards") || JSON::Any.new("{}")
+ return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds)
end
def paid
- reason = info.dig?("playabilityStatus", "reason").try &.as_s || ""
- return reason.includes? "requires payment"
+ return (self.reason || "").includes? "requires payment"
end
def premium
keywords.includes? "YouTube Red"
end
- def captions : Array(Caption)
- return @captions.as(Array(Caption)) if @captions
- captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
- name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
- language_code = caption["languageCode"].to_s
- base_url = caption["baseUrl"].to_s
-
- caption = Caption.new(name.to_s, language_code, base_url)
- caption.name = caption.name.split(" - ")[0]
- caption
+ def captions : Array(Invidious::Videos::Captions::Metadata)
+ if @captions.empty? && @info.has_key?("captions")
+ @captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"])
end
- captions ||= [] of Caption
- @captions = captions
- return @captions.as(Array(Caption))
- end
- def description
- description = info
- .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText")
- .try &.as_s || ""
- end
-
- # TODO
- def description=(value : String)
- @description = value
- end
-
- def description_html
- info["descriptionHtml"]?.try &.as_s || ""
- end
-
- def description_html=(value : String)
- info["descriptionHtml"] = JSON::Any.new(value)
- end
-
- def short_description
- info["shortDescription"]?.try &.as_s? || ""
+ return @captions
end
def hls_manifest_url : String?
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
end
- def dash_manifest_url
- info.dig?("streamingData", "dashManifestUrl").try &.as_s
- end
+ def dash_manifest_url : String?
+ raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s
+ return nil if raw_dash_url.nil?
- def genre : String
- info["genre"]?.try &.as_s || ""
+ # Use manifest v5 parameter to reduce file size
+ # See https://github.com/iv-org/invidious/issues/4186
+ dash_url = URI.parse(raw_dash_url)
+ dash_query = dash_url.query || ""
+
+ if dash_query.empty?
+ dash_url.path = "#{dash_url.path}/mpd_version/5"
+ else
+ dash_url.query = "#{dash_query}&mpd_version=5"
+ end
+
+ return dash_url.to_s
end
def genre_url : String?
- info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
+ info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
end
- def license : String?
- info["license"]?.try &.as_s
- end
-
- def is_family_friendly : Bool
- info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false
- end
-
- def is_vr : Bool?
- projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
- return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
+ def vr? : Bool?
+ return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
end
def projection_type : String?
return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
end
- def wilson_score : Float64
- ci_lower_bound(likes, likes + dislikes).round(4)
- end
-
- def engagement : Float64
- (((likes + dislikes) / views) * 100).round(4)
- end
-
def reason : String?
info["reason"]?.try &.as_s
end
-end
-struct Caption
- property name
- property language_code
- property base_url
-
- getter name : String
- getter language_code : String
- getter base_url : String
-
- setter name
-
- def initialize(@name, @language_code, @base_url)
- end
-end
-
-class VideoRedirect < Exception
- property video_id : String
-
- def initialize(@video_id)
- end
-end
-
-# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
-# The former is preferred as it has more videos in it. The second has
-# the same 11 first entries as the compact rendered.
-#
-# TODO: "compactRadioRenderer" (Mix) and
-def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
- return nil if !related["videoId"]?
-
- # The compact renderer has video length in seconds, where the end
- # screen rendered has a full text version ("42:40")
- length = related["lengthInSeconds"]?.try &.as_i.to_s
- length ||= related.dig?("lengthText", "simpleText").try do |box|
- decode_length_seconds(box.as_s).to_s
+ def music : Array(VideoMusic)
+ info["music"].as_a.map { |music_json|
+ VideoMusic.new(
+ music_json["song"].as_s,
+ music_json["album"].as_s,
+ music_json["artist"].as_s,
+ music_json["license"].as_s
+ )
+ }
end
- # Both have "short", so the "long" option shouldn't be required
- channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
- .try &.dig?("runs", 0)
+ # Macros defining getters/setters for various types of data
- author = channel_info.try &.dig?("text")
- ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
-
- # "4,088,033 views", only available on compact renderer
- # and when video is not a livestream
- view_count = related.dig?("viewCountText", "simpleText")
- .try &.as_s.gsub(/\D/, "")
-
- short_view_count = related.try do |r|
- HelperExtractors.get_short_view_count(r).to_s
- end
-
- LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
-
- # TODO: when refactoring video types, make a struct for related videos
- # or reuse an existing type, if that fits.
- return {
- "id" => related["videoId"],
- "title" => related["title"]["simpleText"],
- "author" => author || JSON::Any.new(""),
- "ucid" => JSON::Any.new(ucid || ""),
- "length_seconds" => JSON::Any.new(length || "0"),
- "view_count" => JSON::Any.new(view_count || "0"),
- "short_view_count" => JSON::Any.new(short_view_count || "0"),
- }
-end
-
-def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil)
- params = {} of String => JSON::Any
-
- client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
- if context_screen == "embed"
- client_config.client_type = YoutubeAPI::ClientType::WebScreenEmbed
- end
-
- player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
-
- if player_response.dig?("playabilityStatus", "status").try &.as_s != "OK"
- subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
- reason = subreason.try &.[]?("simpleText").try &.as_s
- reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
- reason ||= player_response.dig("playabilityStatus", "reason").as_s
- params["reason"] = JSON::Any.new(reason)
- return params
- end
-
- params["shortDescription"] = player_response.dig?("videoDetails", "shortDescription") || JSON::Any.new(nil)
-
- # Don't fetch the next endpoint if the video is unavailable.
- if !params["reason"]?
- next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
- player_response = player_response.merge(next_response)
- end
-
- # Fetch the video streams using an Android client in order to get the decrypted URLs and
- # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs:
- # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
- if !params["reason"]?
- if context_screen == "embed"
- client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
- else
- client_config.client_type = YoutubeAPI::ClientType::Android
+ private macro getset_string(name)
+ # Return {{name.stringify}} from `info`
+ def {{name.id.underscore}} : String
+ return info[{{name.stringify}}]?.try &.as_s || ""
end
- android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
- # Sometime, the video is available from the web client, but not on Android, so check
- # that here, and fallback to the streaming data from the web client if needed.
- # See: https://github.com/iv-org/invidious/issues/2549
- if android_player["playabilityStatus"]["status"] == "OK"
- params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("")
- else
- params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("")
+ # Update {{name.stringify}} into `info`
+ def {{name.id.underscore}}=(value : String)
+ info[{{name.stringify}}] = JSON::Any.new(value)
end
+
+ {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f|
- params[f] = player_response[f] if player_response[f]?
- end
-
- # Top level elements
-
- main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
-
- raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
-
- primary_results = main_results.dig?("results", "results", "contents")
-
- raise BrokenTubeException.new("results") if !primary_results
-
- video_primary_renderer = primary_results
- .as_a.find(&.["videoPrimaryInfoRenderer"]?)
- .try &.["videoPrimaryInfoRenderer"]
-
- video_secondary_renderer = primary_results
- .as_a.find(&.["videoSecondaryInfoRenderer"]?)
- .try &.["videoSecondaryInfoRenderer"]
-
- raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
- raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
-
- # Related videos
-
- LOGGER.debug("extract_video_info: parsing related videos...")
-
- related = [] of JSON::Any
-
- # Parse "compactVideoRenderer" items (under secondary results)
- secondary_results = main_results
- .dig?("secondaryResults", "secondaryResults", "results")
- secondary_results.try &.as_a.each do |element|
- if item = element["compactVideoRenderer"]?
- related_video = parse_related_video(item)
- related << JSON::Any.new(related_video) if related_video
+ private macro getset_string_array(name)
+ # Return {{name.stringify}} from `info`
+ def {{name.id.underscore}} : Array(String)
+ return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String
end
+
+ # Update {{name.stringify}} into `info`
+ def {{name.id.underscore}}=(value : Array(String))
+ info[{{name.stringify}}] = JSON::Any.new(value)
+ end
+
+ {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- # If nothing was found previously, fall back to end screen renderer
- if related.empty?
- # Container for "endScreenVideoRenderer" items
- player_overlays = player_response.dig?(
- "playerOverlays", "playerOverlayRenderer",
- "endScreen", "watchNextEndScreenRenderer", "results"
- )
-
- player_overlays.try &.as_a.each do |element|
- if item = element["endScreenVideoRenderer"]?
- related_video = parse_related_video(item)
- related << JSON::Any.new(related_video) if related_video
+ {% for op, type in {i32: Int32, i64: Int64} %}
+ private macro getset_{{op}}(name)
+ def \{{name.id.underscore}} : {{type}}
+ return info[\{{name.stringify}}]?.try &.as_i64.to_{{op}} || 0_{{op}}
end
+
+ def \{{name.id.underscore}}=(value : Int)
+ info[\{{name.stringify}}] = JSON::Any.new(value.to_i64)
+ end
+
+ \{% if flag?(:debug_macros) %} \{{debug}} \{% end %}
end
+ {% end %}
+
+ private macro getset_bool(name)
+ # Return {{name.stringify}} from `info`
+ def {{name.id.underscore}} : Bool
+ return info[{{name.stringify}}]?.try &.as_bool || false
+ end
+
+ # Update {{name.stringify}} into `info`
+ def {{name.id.underscore}}=(value : Bool)
+ info[{{name.stringify}}] = JSON::Any.new(value)
+ end
+
+ {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- params["relatedVideos"] = JSON::Any.new(related)
-
- # Likes/dislikes
-
- toplevel_buttons = video_primary_renderer
- .try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
-
- if toplevel_buttons
- likes_button = toplevel_buttons.as_a
- .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "LIKE")
- .try &.["toggleButtonRenderer"]
-
- if likes_button
- likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?)
- .try &.dig?("accessibility", "accessibilityData", "label")
- likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
-
- LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
- LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
+ # Macro to generate ? and = accessor methods for attributes in `info`
+ private macro predicate_bool(method_name, name)
+ # Return {{name.stringify}} from `info`
+ def {{method_name.id.underscore}}? : Bool
+ return info[{{name.stringify}}]?.try &.as_bool || false
end
- dislikes_button = toplevel_buttons.as_a
- .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "DISLIKE")
- .try &.["toggleButtonRenderer"]
-
- if dislikes_button
- dislikes_txt = (dislikes_button["defaultText"]? || dislikes_button["toggledText"]?)
- .try &.dig?("accessibility", "accessibilityData", "label")
- dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64? if dislikes_txt
-
- LOGGER.trace("extract_video_info: Found \"dislikes\" button. Button text is \"#{dislikes_txt}\"")
- LOGGER.debug("extract_video_info: Dislikes count is #{dislikes}") if dislikes
+ # Update {{name.stringify}} into `info`
+ def {{method_name.id.underscore}}=(value : Bool)
+ info[{{name.stringify}}] = JSON::Any.new(value)
end
+
+ {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- if likes && likes != 0_i64 && (!dislikes || dislikes == 0_i64)
- if rating = player_response.dig?("videoDetails", "averageRating").try { |x| x.as_i64? || x.as_f? }
- dislikes = (likes * ((5 - rating)/(rating - 1))).round.to_i64
- LOGGER.debug("extract_video_info: Dislikes count (using fallback method) is #{dislikes}")
- end
- end
+ # Method definitions, using the macros above
- params["likes"] = JSON::Any.new(likes || 0_i64)
- params["dislikes"] = JSON::Any.new(dislikes || 0_i64)
+ getset_string author
+ getset_string authorThumbnail
+ getset_string description
+ getset_string descriptionHtml
+ getset_string genre
+ getset_string genreUcid
+ getset_string license
+ getset_string shortDescription
+ getset_string subCountText
+ getset_string title
+ getset_string ucid
- # Description
+ getset_string_array allowedRegions
+ getset_string_array keywords
- description_html = video_secondary_renderer.try &.dig?("description", "runs")
- .try &.as_a.try { |t| content_to_comment_html(t) }
+ getset_i32 lengthSeconds
+ getset_i64 likes
+ getset_i64 views
- params["descriptionHtml"] = JSON::Any.new(description_html || "")
-
- # Video metadata
-
- metadata = video_secondary_renderer
- .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
- .try &.as_a
-
- params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("")
- params["genreUrl"] = JSON::Any.new(nil)
-
- metadata.try &.each do |row|
- title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s
- contents = row.dig?("metadataRowRenderer", "contents", 0)
-
- if title.try &.== "Category"
- contents = contents.try &.dig?("runs", 0)
-
- params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "")
- params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]?
- .try &.["browseId"]?.try &.as_s || "")
- elsif title.try &.== "License"
- contents = contents.try &.["runs"]?
- .try &.as_a[0]?
-
- params["license"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "")
- elsif title.try &.== "Licensed to YouTube by"
- params["license"] = JSON::Any.new(contents.try &.["simpleText"]?.try &.as_s || "")
- end
- end
-
- # Author infos
-
- author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
- author_thumbnail = author_info.try &.dig?("thumbnail", "thumbnails", 0, "url")
-
- params["authorThumbnail"] = JSON::Any.new(author_thumbnail.try &.as_s || "")
-
- params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]?
- .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }.try &.as_s.split(" ", 2)[0] || "-")
-
- # Return data
-
- return params
+ # TODO: Make predicate_bool the default as to adhere to Crystal conventions
+ getset_bool allowRatings
+ getset_bool authorVerified
+ getset_bool isFamilyFriendly
+ getset_bool isListed
+ predicate_bool upcoming, isUpcoming
end
def get_video(id, refresh = true, region = nil, force_refresh = false)
@@ -1079,7 +301,8 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
if (refresh &&
(Time.utc - video.updated > 10.minutes) ||
(video.premiere_timestamp.try &.< Time.utc)) ||
- force_refresh
+ force_refresh ||
+ video.schema_version != Video::SCHEMA_VERSION # cache control
begin
video = fetch_video(id, region)
Invidious::Database::Videos.update(video)
@@ -1094,34 +317,23 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
end
return video
+rescue DB::Error
+ # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
+ # Note: All DB errors inherit from `DB::Error`
+ return fetch_video(id, region)
end
def fetch_video(id, region)
info = extract_video_info(video_id: id)
- allowed_regions = info
- .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
- .try &.as_a.map &.as_s || [] of String
-
- # Check for region-blocks
- if info["reason"]?.try &.as_s.includes?("your country")
- bypass_regions = PROXY_LIST.keys & allowed_regions
- if !bypass_regions.empty?
- region = bypass_regions[rand(bypass_regions.size)]
- region_info = extract_video_info(video_id: id, proxy_region: region)
- region_info["region"] = JSON::Any.new(region) if region
- info = region_info if !region_info["reason"]?
- end
- end
-
- # Try to fetch video info using an embedded client
- if info["reason"]?
- embed_info = extract_video_info(video_id: id, context_screen: "embed")
- info = embed_info if !embed_info["reason"]?
- end
-
if reason = info["reason"]?
- raise InfoException.new(reason.as_s || "")
+ if reason == "Video unavailable"
+ raise NotFoundException.new(reason.as_s || "")
+ elsif !reason.as_s.starts_with? "Premieres"
+ # dont error when it's a premiere.
+ # we already parsed most of the data and display the premiere date
+ raise InfoException.new(reason.as_s || "")
+ end
end
video = Video.new({
@@ -1133,10 +345,6 @@ def fetch_video(id, region)
return video
end
-def itag_to_metadata?(itag : JSON::Any)
- return VIDEO_FORMATS[itag.to_s]?
-end
-
def process_continuation(query, plid, id)
continuation = nil
if plid
@@ -1151,135 +359,6 @@ def process_continuation(query, plid, id)
continuation
end
-def process_video_params(query, preferences)
- annotations = query["iv_load_policy"]?.try &.to_i?
- autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- comments = query["comments"]?.try &.split(",").map(&.downcase)
- continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- player_style = query["player_style"]?
- preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
- quality = query["quality"]?
- quality_dash = query["quality_dash"]?
- region = query["region"]?
- related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- speed = query["speed"]?.try &.rchop("x").to_f?
- video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- volume = query["volume"]?.try &.to_i?
- vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
-
- if preferences
- # region ||= preferences.region
- annotations ||= preferences.annotations.to_unsafe
- autoplay ||= preferences.autoplay.to_unsafe
- comments ||= preferences.comments
- continue ||= preferences.continue.to_unsafe
- continue_autoplay ||= preferences.continue_autoplay.to_unsafe
- listen ||= preferences.listen.to_unsafe
- local ||= preferences.local.to_unsafe
- player_style ||= preferences.player_style
- preferred_captions ||= preferences.captions
- quality ||= preferences.quality
- quality_dash ||= preferences.quality_dash
- related_videos ||= preferences.related_videos.to_unsafe
- speed ||= preferences.speed
- video_loop ||= preferences.video_loop.to_unsafe
- extend_desc ||= preferences.extend_desc.to_unsafe
- volume ||= preferences.volume
- vr_mode ||= preferences.vr_mode.to_unsafe
- save_player_pos ||= preferences.save_player_pos.to_unsafe
- end
-
- annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
- autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
- comments ||= CONFIG.default_user_preferences.comments
- continue ||= CONFIG.default_user_preferences.continue.to_unsafe
- continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
- listen ||= CONFIG.default_user_preferences.listen.to_unsafe
- local ||= CONFIG.default_user_preferences.local.to_unsafe
- player_style ||= CONFIG.default_user_preferences.player_style
- preferred_captions ||= CONFIG.default_user_preferences.captions
- quality ||= CONFIG.default_user_preferences.quality
- quality_dash ||= CONFIG.default_user_preferences.quality_dash
- related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
- speed ||= CONFIG.default_user_preferences.speed
- video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
- extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
- volume ||= CONFIG.default_user_preferences.volume
- vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
- save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
-
- annotations = annotations == 1
- autoplay = autoplay == 1
- continue = continue == 1
- continue_autoplay = continue_autoplay == 1
- listen = listen == 1
- local = local == 1
- related_videos = related_videos == 1
- video_loop = video_loop == 1
- extend_desc = extend_desc == 1
- vr_mode = vr_mode == 1
- save_player_pos = save_player_pos == 1
-
- if CONFIG.disabled?("dash") && quality == "dash"
- quality = "high"
- end
-
- if CONFIG.disabled?("local") && local
- local = false
- end
-
- if start = query["t"]? || query["time_continue"]? || query["start"]?
- video_start = decode_time(start)
- end
- video_start ||= 0
-
- if query["end"]?
- video_end = decode_time(query["end"])
- end
- video_end ||= -1
-
- raw = query["raw"]?.try &.to_i?
- raw ||= 0
- raw = raw == 1
-
- controls = query["controls"]?.try &.to_i?
- controls ||= 1
- controls = controls >= 1
-
- params = VideoPreferences.new({
- annotations: annotations,
- autoplay: autoplay,
- comments: comments,
- continue: continue,
- continue_autoplay: continue_autoplay,
- controls: controls,
- listen: listen,
- local: local,
- player_style: player_style,
- preferred_captions: preferred_captions,
- quality: quality,
- quality_dash: quality_dash,
- raw: raw,
- region: region,
- related_videos: related_videos,
- speed: speed,
- video_end: video_end,
- video_loop: video_loop,
- extend_desc: extend_desc,
- video_start: video_start,
- volume: volume,
- vr_mode: vr_mode,
- save_player_pos: save_player_pos,
- })
-
- return params
-end
-
def build_thumbnails(id)
return {
{host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"},
@@ -1293,34 +372,3 @@ def build_thumbnails(id)
{host: HOST_URL, height: 90, width: 120, name: "end", url: "3"},
}
end
-
-def generate_thumbnails(json, id)
- json.array do
- build_thumbnails(id).each do |thumbnail|
- json.object do
- json.field "quality", thumbnail[:name]
- json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
- json.field "width", thumbnail[:width]
- json.field "height", thumbnail[:height]
- end
- end
- end
-end
-
-def generate_storyboards(json, id, storyboards)
- json.array do
- storyboards.each do |storyboard|
- json.object do
- json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}"
- json.field "templateUrl", storyboard[:url]
- json.field "width", storyboard[:width]
- json.field "height", storyboard[:height]
- json.field "count", storyboard[:count]
- json.field "interval", storyboard[:interval]
- json.field "storyboardWidth", storyboard[:storyboard_width]
- json.field "storyboardHeight", storyboard[:storyboard_height]
- json.field "storyboardCount", storyboard[:storyboard_count]
- end
- end
- end
-end
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
new file mode 100644
index 00000000..c811cfe1
--- /dev/null
+++ b/src/invidious/videos/caption.cr
@@ -0,0 +1,224 @@
+require "json"
+
+module Invidious::Videos
+ module Captions
+ struct Metadata
+ property name : String
+ property language_code : String
+ property base_url : String
+
+ property auto_generated : Bool
+
+ def initialize(@name, @language_code, @base_url, @auto_generated)
+ end
+
+ # Parse the JSON structure from Youtube
+ def self.from_yt_json(container : JSON::Any) : Array(Captions::Metadata)
+ caption_tracks = container
+ .dig?("playerCaptionsTracklistRenderer", "captionTracks")
+ .try &.as_a
+
+ captions_list = [] of Captions::Metadata
+ return captions_list if caption_tracks.nil?
+
+ caption_tracks.each do |caption|
+ name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
+ name = name.to_s.split(" - ")[0]
+
+ language_code = caption["languageCode"].to_s
+ base_url = caption["baseUrl"].to_s
+
+ auto_generated = (caption["kind"]? == "asr")
+
+ captions_list << Captions::Metadata.new(name, language_code, base_url, auto_generated)
+ end
+
+ return captions_list
+ end
+
+ def timedtext_to_vtt(timedtext : String, tlang = nil) : String
+ # In the future, we could just directly work with the url. This is more of a POC
+ cues = [] of XML::Node
+ tree = XML.parse(timedtext)
+ tree = tree.children.first
+
+ tree.children.each do |item|
+ if item.name == "body"
+ item.children.each do |cue|
+ if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n")
+ cues << cue
+ end
+ end
+ break
+ end
+ end
+
+ settings_field = {
+ "Kind" => "captions",
+ "Language" => "#{tlang || @language_code}",
+ }
+
+ result = WebVTT.build(settings_field) do |vtt|
+ cues.each_with_index do |node, i|
+ start_time = node["t"].to_f.milliseconds
+
+ duration = node["d"]?.try &.to_f.milliseconds
+
+ duration ||= start_time
+
+ if cues.size > i + 1
+ end_time = cues[i + 1]["t"].to_f.milliseconds
+ else
+ end_time = start_time + duration
+ end
+
+ text = String.build do |io|
+ node.children.each do |s|
+ io << s.content
+ end
+ end
+
+ vtt.cue(start_time, end_time, text)
+ end
+ end
+
+ return result
+ end
+ end
+
+ # List of all caption languages available on Youtube.
+ LANGUAGES = {
+ "",
+ "English",
+ "English (auto-generated)",
+ "English (United Kingdom)",
+ "English (United States)",
+ "Afrikaans",
+ "Albanian",
+ "Amharic",
+ "Arabic",
+ "Armenian",
+ "Azerbaijani",
+ "Bangla",
+ "Basque",
+ "Belarusian",
+ "Bosnian",
+ "Bulgarian",
+ "Burmese",
+ "Cantonese (Hong Kong)",
+ "Catalan",
+ "Cebuano",
+ "Chinese",
+ "Chinese (China)",
+ "Chinese (Hong Kong)",
+ "Chinese (Simplified)",
+ "Chinese (Taiwan)",
+ "Chinese (Traditional)",
+ "Corsican",
+ "Croatian",
+ "Czech",
+ "Danish",
+ "Dutch",
+ "Dutch (auto-generated)",
+ "Esperanto",
+ "Estonian",
+ "Filipino",
+ "Filipino (auto-generated)",
+ "Finnish",
+ "French",
+ "French (auto-generated)",
+ "Galician",
+ "Georgian",
+ "German",
+ "German (auto-generated)",
+ "Greek",
+ "Gujarati",
+ "Haitian Creole",
+ "Hausa",
+ "Hawaiian",
+ "Hebrew",
+ "Hindi",
+ "Hmong",
+ "Hungarian",
+ "Icelandic",
+ "Igbo",
+ "Indonesian",
+ "Indonesian (auto-generated)",
+ "Interlingue",
+ "Irish",
+ "Italian",
+ "Italian (auto-generated)",
+ "Japanese",
+ "Japanese (auto-generated)",
+ "Javanese",
+ "Kannada",
+ "Kazakh",
+ "Khmer",
+ "Korean",
+ "Korean (auto-generated)",
+ "Kurdish",
+ "Kyrgyz",
+ "Lao",
+ "Latin",
+ "Latvian",
+ "Lithuanian",
+ "Luxembourgish",
+ "Macedonian",
+ "Malagasy",
+ "Malay",
+ "Malayalam",
+ "Maltese",
+ "Maori",
+ "Marathi",
+ "Mongolian",
+ "Nepali",
+ "Norwegian Bokmål",
+ "Nyanja",
+ "Pashto",
+ "Persian",
+ "Polish",
+ "Portuguese",
+ "Portuguese (auto-generated)",
+ "Portuguese (Brazil)",
+ "Punjabi",
+ "Romanian",
+ "Russian",
+ "Russian (auto-generated)",
+ "Samoan",
+ "Scottish Gaelic",
+ "Serbian",
+ "Shona",
+ "Sindhi",
+ "Sinhala",
+ "Slovak",
+ "Slovenian",
+ "Somali",
+ "Southern Sotho",
+ "Spanish",
+ "Spanish (auto-generated)",
+ "Spanish (Latin America)",
+ "Spanish (Mexico)",
+ "Spanish (Spain)",
+ "Sundanese",
+ "Swahili",
+ "Swedish",
+ "Tajik",
+ "Tamil",
+ "Telugu",
+ "Thai",
+ "Turkish",
+ "Turkish (auto-generated)",
+ "Ukrainian",
+ "Urdu",
+ "Uzbek",
+ "Vietnamese",
+ "Vietnamese (auto-generated)",
+ "Welsh",
+ "Western Frisian",
+ "Xhosa",
+ "Yiddish",
+ "Yoruba",
+ "Zulu",
+ }
+ end
+end
diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr
new file mode 100644
index 00000000..29c57182
--- /dev/null
+++ b/src/invidious/videos/clip.cr
@@ -0,0 +1,22 @@
+require "json"
+
+# returns start_time, end_time and clip_title
+def parse_clip_parameters(params) : {Float64?, Float64?, String?}
+ decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
+ .try { |i| Base64.decode(i) }
+ .try { |i| IO::Memory.new(i) }
+ .try { |i| Protodec::Any.parse(i) }
+
+ start_time = decoded_protobuf
+ .try(&.["50:0:embedded"]["2:1:varint"].as_i64)
+ .try { |i| i/1000 }
+
+ end_time = decoded_protobuf
+ .try(&.["50:0:embedded"]["3:2:varint"].as_i64)
+ .try { |i| i/1000 }
+
+ clip_title = decoded_protobuf
+ .try(&.["50:0:embedded"]["4:3:string"].as_s)
+
+ return start_time, end_time, clip_title
+end
diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr
new file mode 100644
index 00000000..1371bebb
--- /dev/null
+++ b/src/invidious/videos/description.cr
@@ -0,0 +1,82 @@
+require "json"
+require "uri"
+
+private def copy_string(str : String::Builder, iter : Iterator, count : Int) : Int
+ copied = 0
+ while copied < count
+ cp = iter.next
+ break if cp.is_a?(Iterator::Stop)
+
+ if cp == 0x26 # Ampersand (&)
+ str << "&"
+ elsif cp == 0x27 # Single quote (')
+ str << "'"
+ elsif cp == 0x22 # Double quote (")
+ str << """
+ elsif cp == 0x3C # Less-than (<)
+ str << "<"
+ elsif cp == 0x3E # Greater than (>)
+ str << ">"
+ else
+ str << cp.chr
+ end
+
+ # A codepoint from the SMP counts twice
+ copied += 1 if cp > 0xFFFF
+ copied += 1
+ end
+
+ return copied
+end
+
+def parse_description(desc, video_id : String) : String?
+ return "" if desc.nil?
+
+ content = desc["content"].as_s
+ return "" if content.empty?
+
+ commands = desc["commandRuns"]?.try &.as_a
+ if commands.nil?
+ # Slightly faster than HTML.escape, as we're only doing one pass on
+ # the string instead of five for the standard library
+ return String.build do |str|
+ copy_string(str, content.each_codepoint, content.size)
+ end
+ end
+
+ # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints
+ # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are
+ # automatically decoded by the JSON parser. It means that we need to count
+ # copied byte in a special manner, preventing the use of regular string copy.
+ iter = content.each_codepoint
+
+ index = 0
+
+ return String.build do |str|
+ commands.each do |command|
+ cmd_start = command["startIndex"].as_i
+ cmd_length = command["length"].as_i
+
+ # Copy the text chunk between this command and the previous if needed.
+ length = cmd_start - index
+ index += copy_string(str, iter, length)
+
+ # We need to copy the command's text using the iterator
+ # and the special function defined above.
+ cmd_content = String.build(cmd_length) do |str2|
+ copy_string(str2, iter, cmd_length)
+ end
+
+ link = cmd_content
+ if on_tap = command.dig?("onTap", "innertubeCommand")
+ link = parse_link_endpoint(on_tap, cmd_content, video_id)
+ end
+ str << link
+ index += cmd_length
+ end
+
+ # Copy the end of the string (past the last command).
+ remaining_length = content.size - index
+ copy_string(str, iter, remaining_length) if remaining_length > 0
+ end
+end
diff --git a/src/invidious/videos/formats.cr b/src/invidious/videos/formats.cr
new file mode 100644
index 00000000..e98e7257
--- /dev/null
+++ b/src/invidious/videos/formats.cr
@@ -0,0 +1,116 @@
+module Invidious::Videos::Formats
+ def self.itag_to_metadata?(itag : JSON::Any)
+ return FORMATS[itag.to_s]?
+ end
+
+ # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
+ private FORMATS = {
+ "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
+ "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
+ "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
+ "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
+ "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
+ "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+
+ "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
+ "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+
+ # 3D videos
+ "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+
+ # Apple HTTP Live Streaming
+ "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
+ "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
+ "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
+
+ # DASH mp4 video
+ "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
+ "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
+ "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
+ "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
+ "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
+ "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
+ "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
+ "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
+ "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
+ "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
+ "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
+ "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
+
+ # Dash mp4 audio
+ "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
+ "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
+ "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
+ "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
+ "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
+ "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
+ "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
+
+ # Dash webm
+ "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
+ "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
+ "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
+ "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
+ "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
+ "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
+ # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
+ "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
+ "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
+ "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+
+ # Dash webm audio
+ "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
+ "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
+
+ # Dash webm audio with opus inside
+ "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
+ "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
+ "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
+
+ # av01 video only formats sometimes served with "unknown" codecs
+ "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
+ "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
+ "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
+ "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
+ }
+end
diff --git a/src/invidious/videos/music.cr b/src/invidious/videos/music.cr
new file mode 100644
index 00000000..08d88a3e
--- /dev/null
+++ b/src/invidious/videos/music.cr
@@ -0,0 +1,13 @@
+require "json"
+
+struct VideoMusic
+ include JSON::Serializable
+
+ property song : String
+ property album : String
+ property artist : String
+ property license : String
+
+ def initialize(@song : String, @album : String, @artist : String, @license : String)
+ end
+end
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
new file mode 100644
index 00000000..feb58440
--- /dev/null
+++ b/src/invidious/videos/parser.cr
@@ -0,0 +1,504 @@
+require "json"
+
+# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
+# The former is preferred as it has more videos in it. The second has
+# the same 11 first entries as the compact rendered.
+#
+# TODO: "compactRadioRenderer" (Mix) and
+# TODO: Use a proper struct/class instead of a hacky JSON object
+def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
+ return nil if !related["videoId"]?
+
+ # The compact renderer has video length in seconds, where the end
+ # screen rendered has a full text version ("42:40")
+ length = related["lengthInSeconds"]?.try &.as_i.to_s
+ length ||= related.dig?("lengthText", "simpleText").try do |box|
+ decode_length_seconds(box.as_s).to_s
+ end
+
+ # Both have "short", so the "long" option shouldn't be required
+ channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
+ .try &.dig?("runs", 0)
+
+ author = channel_info.try &.dig?("text")
+ author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
+
+ ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
+
+ # "4,088,033 views", only available on compact renderer
+ # and when video is not a livestream
+ view_count = related.dig?("viewCountText", "simpleText")
+ .try &.as_s.gsub(/\D/, "")
+
+ short_view_count = related.try do |r|
+ HelperExtractors.get_short_view_count(r).to_s
+ end
+
+ LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
+
+ if published_time_text = related["publishedTimeText"]?
+ decoded_time = decode_date(published_time_text["simpleText"].to_s)
+ published = decoded_time.to_rfc3339.to_s
+ else
+ published = nil
+ end
+
+ # TODO: when refactoring video types, make a struct for related videos
+ # or reuse an existing type, if that fits.
+ return {
+ "id" => related["videoId"],
+ "title" => related["title"]["simpleText"],
+ "author" => author || JSON::Any.new(""),
+ "ucid" => JSON::Any.new(ucid || ""),
+ "length_seconds" => JSON::Any.new(length || "0"),
+ "view_count" => JSON::Any.new(view_count || "0"),
+ "short_view_count" => JSON::Any.new(short_view_count || "0"),
+ "author_verified" => JSON::Any.new(author_verified),
+ "published" => JSON::Any.new(published || ""),
+ }
+end
+
+def extract_video_info(video_id : String)
+ # Init client config for the API
+ client_config = YoutubeAPI::ClientConfig.new
+
+ # Fetch data from the player endpoint
+ player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
+
+ playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
+
+ if playability_status != "OK"
+ subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
+ reason = subreason.try &.[]?("simpleText").try &.as_s
+ reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
+ reason ||= player_response.dig("playabilityStatus", "reason").as_s
+
+ # Stop here if video is not a scheduled livestream or
+ # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
+ if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
+ playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
+ return {
+ "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+ "reason" => JSON::Any.new(reason),
+ }
+ end
+ elsif video_id != player_response.dig?("videoDetails", "videoId")
+ # YouTube may return a different video player response than expected.
+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+ # Line to be reverted if one day we solve the video not available issue.
+
+ # Although technically not a call to /videoplayback the fact that YouTube is returning the
+ # wrong video means that we should count it as a failure.
+ get_playback_statistic()["totalRequests"] += 1
+
+ return {
+ "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+ "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."),
+ }
+ else
+ reason = nil
+ end
+
+ # Don't fetch the next endpoint if the video is unavailable.
+ if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
+ next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
+ player_response = player_response.merge(next_response)
+ end
+
+ params = parse_video_info(video_id, player_response)
+ params["reason"] = JSON::Any.new(reason) if reason
+
+ if !CONFIG.invidious_companion.present?
+ if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
+ LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
+ players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile}
+
+ players_fallback.each do |player_fallback|
+ client_config.client_type = player_fallback
+
+ next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
+
+ if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
+ streaming_data = player_response["streamingData"].as_h
+ streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
+ player_response["streamingData"] = JSON::Any.new(streaming_data)
+ break
+ end
+ rescue InfoException
+ next LOGGER.warn("Failed to fetch streams with #{player_fallback}")
+ end
+ end
+
+ # Seems like video page can still render even without playable streams.
+ # its better than nothing.
+ #
+ # # Were we able to find playable video streams?
+ # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
+ # # No :(
+ # end
+ end
+
+ {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
+ params[f] = player_response[f] if player_response[f]?
+ end
+
+ # Convert URLs, if those are present
+ if streaming_data = player_response["streamingData"]?
+ %w[formats adaptiveFormats].each do |key|
+ streaming_data.as_h[key]?.try &.as_a.each do |format|
+ format.as_h["url"] = JSON::Any.new(convert_url(format))
+ end
+ end
+
+ params["streamingData"] = streaming_data
+ end
+
+ # Data structure version, for cache control
+ params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
+
+ return params
+end
+
+def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
+ LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
+ response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
+
+ playability_status = response["playabilityStatus"]["status"]
+ LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
+
+ if id != response.dig?("videoDetails", "videoId")
+ # YouTube may return a different video player response than expected.
+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+ raise InfoException.new(
+ "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
+ )
+ elsif playability_status == "OK"
+ return response
+ else
+ return nil
+ end
+end
+
+def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
+ # Top level elements
+
+ main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
+
+ raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
+
+ # Primary results are not available on Music videos
+ # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725
+ if primary_results = main_results.dig?("results", "results", "contents")
+ video_primary_renderer = primary_results
+ .as_a.find(&.["videoPrimaryInfoRenderer"]?)
+ .try &.["videoPrimaryInfoRenderer"]
+
+ video_secondary_renderer = primary_results
+ .as_a.find(&.["videoSecondaryInfoRenderer"]?)
+ .try &.["videoSecondaryInfoRenderer"]
+
+ raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
+ raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
+ end
+
+ video_details = player_response.dig?("videoDetails")
+ if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
+ microformat = {} of String => JSON::Any
+ end
+
+ raise BrokenTubeException.new("videoDetails") if !video_details
+
+ # Basic video infos
+
+ title = video_details["title"]?.try &.as_s
+
+ # We have to try to extract viewCount from videoPrimaryInfoRenderer first,
+ # then from videoDetails, as the latter is "0" for livestreams (we want
+ # to get the amount of viewers watching).
+ views_txt = extract_text(
+ video_primary_renderer
+ .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount")
+ )
+ views_txt ||= video_details["viewCount"]?.try &.as_s || ""
+ views = views_txt.gsub(/\D/, "").to_i64?
+
+ length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
+ .try &.as_s.to_i64
+
+ published = microformat["publishDate"]?
+ .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
+
+ premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
+ .try { |t| Time.parse_rfc3339(t.as_s) }
+
+ premiere_timestamp ||= player_response.dig?(
+ "playabilityStatus", "liveStreamability",
+ "liveStreamabilityRenderer", "offlineSlate",
+ "liveStreamOfflineSlateRenderer", "scheduledStartTime"
+ )
+ .try &.as_s.to_i64
+ .try { |t| Time.unix(t) }
+
+ live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
+ .try &.as_bool
+ live_now ||= video_details.dig?("isLive").try &.as_bool || false
+
+ post_live_dvr = video_details.dig?("isPostLiveDvr")
+ .try &.as_bool || false
+
+ # Extra video infos
+
+ allowed_regions = microformat["availableCountries"]?
+ .try &.as_a.map &.as_s || [] of String
+
+ allow_ratings = video_details["allowRatings"]?.try &.as_bool
+ family_friendly = microformat["isFamilySafe"]?.try &.as_bool
+ is_listed = video_details["isCrawlable"]?.try &.as_bool
+ is_upcoming = video_details["isUpcoming"]?.try &.as_bool
+
+ keywords = video_details["keywords"]?
+ .try &.as_a.map &.as_s || [] of String
+
+ # Related videos
+
+ LOGGER.debug("extract_video_info: parsing related videos...")
+
+ related = [] of JSON::Any
+
+ # Parse "compactVideoRenderer" items (under secondary results)
+ secondary_results = main_results
+ .dig?("secondaryResults", "secondaryResults", "results")
+ secondary_results.try &.as_a.each do |element|
+ if item = element["compactVideoRenderer"]?
+ related_video = parse_related_video(item)
+ related << JSON::Any.new(related_video) if related_video
+ end
+ end
+
+ # If nothing was found previously, fall back to end screen renderer
+ if related.empty?
+ # Container for "endScreenVideoRenderer" items
+ player_overlays = player_response.dig?(
+ "playerOverlays", "playerOverlayRenderer",
+ "endScreen", "watchNextEndScreenRenderer", "results"
+ )
+
+ player_overlays.try &.as_a.each do |element|
+ if item = element["endScreenVideoRenderer"]?
+ related_video = parse_related_video(item)
+ related << JSON::Any.new(related_video) if related_video
+ end
+ end
+ end
+
+ # Likes
+
+ toplevel_buttons = video_primary_renderer
+ .try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
+
+ if toplevel_buttons
+ # New Format as of december 2023
+ likes_button = toplevel_buttons.dig?(0,
+ "segmentedLikeDislikeButtonViewModel",
+ "likeButtonViewModel",
+ "likeButtonViewModel",
+ "toggleButtonViewModel",
+ "toggleButtonViewModel",
+ "defaultButtonViewModel",
+ "buttonViewModel"
+ )
+
+ likes_button ||= toplevel_buttons.try &.as_a
+ .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE")
+ .try &.["toggleButtonRenderer"]
+
+ # New format as of september 2022
+ likes_button ||= toplevel_buttons.try &.as_a
+ .find(&.["segmentedLikeDislikeButtonRenderer"]?)
+ .try &.dig?(
+ "segmentedLikeDislikeButtonRenderer",
+ "likeButton", "toggleButtonRenderer"
+ )
+
+ if likes_button
+ likes_txt = likes_button.dig?("accessibilityText")
+ # Note: The like count from `toggledText` is off by one, as it would
+ # represent the new like count in the event where the user clicks on "like".
+ likes_txt ||= (likes_button["defaultText"]? || likes_button["toggledText"]?)
+ .try &.dig?("accessibility", "accessibilityData", "label")
+ likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
+
+ LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
+ LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
+ end
+ end
+
+ # Description
+
+ description = microformat.dig?("description", "simpleText").try &.as_s || ""
+ short_description = player_response.dig?("videoDetails", "shortDescription")
+
+ # description_html = video_secondary_renderer.try &.dig?("description", "runs")
+ # .try &.as_a.try { |t| content_to_comment_html(t, video_id) }
+
+ description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"), video_id)
+
+ # Video metadata
+
+ metadata = video_secondary_renderer
+ .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
+ .try &.as_a
+
+ genre = microformat["category"]?
+ genre_ucid = nil
+ license = nil
+
+ metadata.try &.each do |row|
+ metadata_title = extract_text(row.dig?("metadataRowRenderer", "title"))
+ contents = row.dig?("metadataRowRenderer", "contents", 0)
+
+ if metadata_title == "Category"
+ contents = contents.try &.dig?("runs", 0)
+
+ genre = contents.try &.["text"]?
+ genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId")
+ elsif metadata_title == "License"
+ license = contents.try &.dig?("runs", 0, "text")
+ elsif metadata_title == "Licensed to YouTube by"
+ license = contents.try &.["simpleText"]?
+ end
+ end
+
+ # Music section
+
+ music_list = [] of VideoMusic
+ music_desclist = player_response.dig?(
+ "engagementPanels", 1, "engagementPanelSectionListRenderer",
+ "content", "structuredDescriptionContentRenderer", "items", 2,
+ "videoDescriptionMusicSectionRenderer", "carouselLockups"
+ )
+
+ music_desclist.try &.as_a.each do |music_desc|
+ artist = nil
+ album = nil
+ music_license = nil
+
+ # Used when the video has multiple songs
+ if song_title = music_desc.dig?("carouselLockupRenderer", "videoLockup", "compactVideoRenderer", "title")
+ # "simpleText" for plain text / "runs" when song has a link
+ song = song_title["simpleText"]? || song_title.dig?("runs", 0, "text")
+
+ # some videos can have empty tracks. See: https://www.youtube.com/watch?v=eBGIQ7ZuuiU
+ next if !song
+ end
+
+ music_desc.dig?("carouselLockupRenderer", "infoRows").try &.as_a.each do |desc|
+ desc_title = extract_text(desc.dig?("infoRowRenderer", "title"))
+ if desc_title == "ARTIST"
+ artist = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
+ elsif desc_title == "SONG"
+ song = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
+ elsif desc_title == "ALBUM"
+ album = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
+ elsif desc_title == "LICENSES"
+ music_license = extract_text(desc.dig?("infoRowRenderer", "expandedMetadata"))
+ end
+ end
+ music_list << VideoMusic.new(song.to_s, album.to_s, artist.to_s, music_license.to_s)
+ end
+
+ # Author infos
+
+ author = video_details["author"]?.try &.as_s
+ ucid = video_details["channelId"]?.try &.as_s
+
+ if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
+ author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
+ author_verified = has_verified_badge?(author_info["badges"]?)
+
+ subs_text = author_info["subscriberCountText"]?
+ .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
+ .try &.as_s.split(" ", 2)[0]
+ end
+
+ # Return data
+
+ if live_now
+ video_type = VideoType::Livestream
+ elsif !premiere_timestamp.nil?
+ video_type = VideoType::Scheduled
+ published = premiere_timestamp || Time.utc
+ else
+ video_type = VideoType::Video
+ end
+
+ params = {
+ "videoType" => JSON::Any.new(video_type.to_s),
+ # Basic video infos
+ "title" => JSON::Any.new(title || ""),
+ "views" => JSON::Any.new(views || 0_i64),
+ "likes" => JSON::Any.new(likes || 0_i64),
+ "lengthSeconds" => JSON::Any.new(length_txt || 0_i64),
+ "published" => JSON::Any.new(published.to_rfc3339),
+ # Extra video infos
+ "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
+ "allowRatings" => JSON::Any.new(allow_ratings || false),
+ "isFamilyFriendly" => JSON::Any.new(family_friendly || false),
+ "isListed" => JSON::Any.new(is_listed || false),
+ "isUpcoming" => JSON::Any.new(is_upcoming || false),
+ "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
+ "isPostLiveDvr" => JSON::Any.new(post_live_dvr),
+ # Related videos
+ "relatedVideos" => JSON::Any.new(related),
+ # Description
+ "description" => JSON::Any.new(description || ""),
+ "descriptionHtml" => JSON::Any.new(description_html || ""),
+ "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+ # Video metadata
+ "genre" => JSON::Any.new(genre.try &.as_s || ""),
+ "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
+ "license" => JSON::Any.new(license.try &.as_s || ""),
+ # Music section
+ "music" => JSON.parse(music_list.to_json),
+ # Author infos
+ "author" => JSON::Any.new(author || ""),
+ "ucid" => JSON::Any.new(ucid || ""),
+ "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
+ "authorVerified" => JSON::Any.new(author_verified || false),
+ "subCountText" => JSON::Any.new(subs_text || "-"),
+ }
+
+ return params
+end
+
+private def convert_url(fmt)
+ if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
+ sp = cfr["sp"]
+ url = URI.parse(cfr["url"])
+ params = url.query_params
+
+ LOGGER.debug("convert_url: Decoding '#{cfr}'")
+
+ unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
+ params[sp] = unsig if unsig
+ else
+ url = URI.parse(fmt["url"].as_s)
+ params = url.query_params
+ end
+
+ n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
+ params["n"] = n if n
+
+ if token = CONFIG.po_token
+ params["pot"] = token
+ end
+
+ url.query_params = params
+ LOGGER.trace("convert_url: new url is '#{url}'")
+
+ return url.to_s
+rescue ex
+ LOGGER.debug("convert_url: Error when parsing video URL")
+ LOGGER.trace(ex.inspect_with_backtrace)
+ return ""
+end
diff --git a/src/invidious/videos/regions.cr b/src/invidious/videos/regions.cr
new file mode 100644
index 00000000..575f8c25
--- /dev/null
+++ b/src/invidious/videos/regions.cr
@@ -0,0 +1,27 @@
+# List of geographical regions that Youtube recognizes.
+# This is used to determine if a video is either restricted to a list
+# of allowed regions (= whitelisted) or if it can't be watched in
+# a set of regions (= blacklisted).
+REGIONS = {
+ "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT",
+ "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI",
+ "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY",
+ "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN",
+ "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM",
+ "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK",
+ "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL",
+ "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM",
+ "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR",
+ "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN",
+ "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS",
+ "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK",
+ "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW",
+ "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP",
+ "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM",
+ "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW",
+ "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM",
+ "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF",
+ "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW",
+ "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI",
+ "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW",
+}
diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr
new file mode 100644
index 00000000..bd0eef59
--- /dev/null
+++ b/src/invidious/videos/storyboard.cr
@@ -0,0 +1,122 @@
+require "uri"
+require "http/params"
+
+module Invidious::Videos
+ struct Storyboard
+ # Template URL
+ getter url : URI
+ getter proxied_url : URI
+
+ # Thumbnail parameters
+ getter width : Int32
+ getter height : Int32
+ getter count : Int32
+ getter interval : Int32
+
+ # Image (storyboard) parameters
+ getter rows : Int32
+ getter columns : Int32
+ getter images_count : Int32
+
+ def initialize(
+ *, @url, @width, @height, @count, @interval,
+ @rows, @columns, @images_count,
+ )
+ authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]?
+
+ @proxied_url = URI.parse(HOST_URL)
+ @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}"
+ @proxied_url.query = @url.query
+ end
+
+ # Parse the JSON structure from Youtube
+ def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard)
+ # Livestream storyboards are a bit different
+ # TODO: document exactly how
+ if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s
+ return [Storyboard.new(
+ url: URI.parse(storyboard.split("#")[0]),
+ width: 106,
+ height: 60,
+ count: -1,
+ interval: 5000,
+ rows: 3,
+ columns: 3,
+ images_count: -1
+ )]
+ end
+
+ # Split the storyboard string into chunks
+ #
+ # General format (whitespaces added for legibility):
+ # https://i.ytimg.com/sb//storyboard3_L$L/$N.jpg?sqp=
+ # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$
+ # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$
+ # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$
+ #
+ storyboards = container.dig?("playerStoryboardSpecRenderer", "spec")
+ .try &.as_s.split("|")
+
+ return [] of Storyboard if !storyboards
+
+ # The base URL is the first chunk
+ base_url = URI.parse(storyboards.shift)
+
+ return storyboards.map_with_index do |sb, i|
+ # Separate the different storyboard parameters:
+ # width/height: respective dimensions, in pixels, of a single thumbnail
+ # count: how many thumbnails are displayed across the full video
+ # columns/rows: maximum amount of thumbnails that can be stuffed in a
+ # single image, horizontally and vertically.
+ # interval: interval between two thumbnails, in milliseconds
+ # name: storyboard filename. Usually "M$M" or "default"
+ # sigh: URL cryptographic signature
+ width, height, count, columns, rows, interval, name, sigh = sb.split("#")
+
+ width = width.to_i
+ height = height.to_i
+ count = count.to_i
+ interval = interval.to_i
+ columns = columns.to_i
+ rows = rows.to_i
+
+ # Copy base URL object, so that we can modify it
+ url = base_url.dup
+
+ # Add the signature to the URL
+ params = url.query_params
+ params["sigh"] = sigh
+ url.query_params = params
+
+ # Replace the template parts with what we have
+ url.path = url.path.sub("$L", i).sub("$N", name)
+
+ # This value represents the maximum amount of thumbnails that can fit
+ # in a single image. The last image (or the only one for short videos)
+ # will contain less thumbnails than that.
+ thumbnails_per_image = columns * rows
+
+ # This value represents the total amount of storyboards required to
+ # hold all of the thumbnails. It can't be less than 1.
+ images_count = (count / thumbnails_per_image).ceil.to_i
+
+ # Compute the interval when needed (in general, that's only required
+ # for the first "default" storyboard).
+ if interval == 0
+ interval = ((length_seconds / count) * 1_000).to_i
+ end
+
+ Storyboard.new(
+ url: url,
+ width: width,
+ height: height,
+ count: count,
+ interval: interval,
+ rows: rows,
+ columns: columns,
+ images_count: images_count,
+ )
+ end
+ end
+ end
+end
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
new file mode 100644
index 00000000..ee1272d1
--- /dev/null
+++ b/src/invidious/videos/transcript.cr
@@ -0,0 +1,161 @@
+module Invidious::Videos
+ # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video.
+ # These lines can be categorized into two types: section headings and regular lines representing content from the video.
+ struct Transcript
+ # Types
+ record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+ record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+ alias TranscriptLine = HeadingLine | RegularLine
+
+ property lines : Array(TranscriptLine)
+
+ property language_code : String
+ property auto_generated : Bool
+
+ # User friendly label for the current transcript.
+ # Example: "English (auto-generated)"
+ property label : String
+
+ # Initializes a new Transcript struct with the contents and associated metadata describing it
+ def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String)
+ end
+
+ # Generates a protobuf string to fetch the requested transcript from YouTube
+ def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
+ kind = auto_generated ? "asr" : ""
+
+ object = {
+ "1:0:string" => video_id,
+
+ "2:base64" => {
+ "1:string" => kind,
+ "2:string" => language_code,
+ "3:string" => "",
+ },
+
+ "3:varint" => 1_i64,
+ "5:string" => "engagement-panel-searchable-transcript-search-panel",
+ "6:varint" => 1_i64,
+ "7:varint" => 1_i64,
+ "8:varint" => 1_i64,
+ }
+
+ params = object.try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+
+ return params
+ end
+
+ # Constructs a Transcripts struct from the initial YouTube response
+ def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool)
+ transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
+ "content", "transcriptSearchPanelRenderer")
+
+ segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer")
+
+ if !segment_list["initialSegments"]?
+ raise NotFoundException.new("Requested transcript does not exist")
+ end
+
+ # Extract user-friendly label for the current transcript
+
+ footer_language_menu = transcript_panel.dig?(
+ "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems"
+ )
+
+ if footer_language_menu
+ label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
+ else
+ label = language_code
+ end
+
+ # Extract transcript lines
+
+ initial_segments = segment_list["initialSegments"].as_a
+
+ lines = [] of TranscriptLine
+
+ initial_segments.each do |line|
+ if unpacked_line = line["transcriptSectionHeaderRenderer"]?
+ line_type = HeadingLine
+ else
+ unpacked_line = line["transcriptSegmentRenderer"]
+ line_type = RegularLine
+ end
+
+ start_ms = unpacked_line["startMs"].as_s.to_i.millisecond
+ end_ms = unpacked_line["endMs"].as_s.to_i.millisecond
+ text = extract_text(unpacked_line["snippet"]) || ""
+
+ lines << line_type.new(start_ms, end_ms, text)
+ end
+
+ return Transcript.new(
+ lines: lines,
+ language_code: language_code,
+ auto_generated: auto_generated,
+ label: label
+ )
+ end
+
+ # Converts transcript lines to a WebVTT file
+ #
+ # This is used within Invidious to replace subtitles
+ # as to workaround YouTube's rate-limited timedtext endpoint.
+ def to_vtt
+ settings_field = {
+ "Kind" => "captions",
+ "Language" => @language_code,
+ }
+
+ vtt = WebVTT.build(settings_field) do |builder|
+ @lines.each do |line|
+ # Section headers are excluded from the VTT conversion as to
+ # match the regular captions returned from YouTube as much as possible
+ next if line.is_a? HeadingLine
+
+ builder.cue(line.start_ms, line.end_ms, line.line)
+ end
+ end
+
+ return vtt
+ end
+
+ def to_json(json : JSON::Builder)
+ json.field "languageCode", @language_code
+ json.field "autoGenerated", @auto_generated
+ json.field "label", @label
+ json.field "body" do
+ json.array do
+ @lines.each do |line|
+ json.object do
+ if line.is_a? HeadingLine
+ json.field "type", "heading"
+ else
+ json.field "type", "regular"
+ end
+
+ json.field "startMs", line.start_ms.total_milliseconds
+ json.field "endMs", line.end_ms.total_milliseconds
+ json.field "line", line.line
+ end
+ end
+ end
+ end
+ end
+
+ def to_json
+ JSON.build do |json|
+ json.object do
+ json.field "transcript" do
+ json.object do
+ to_json(json)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr
new file mode 100644
index 00000000..48177bd8
--- /dev/null
+++ b/src/invidious/videos/video_preferences.cr
@@ -0,0 +1,162 @@
+struct VideoPreferences
+ include JSON::Serializable
+
+ property annotations : Bool
+ property preload : Bool
+ property autoplay : Bool
+ property comments : Array(String)
+ property continue : Bool
+ property continue_autoplay : Bool
+ property controls : Bool
+ property listen : Bool
+ property local : Bool
+ property preferred_captions : Array(String)
+ property player_style : String
+ property quality : String
+ property quality_dash : String
+ property raw : Bool
+ property region : String?
+ property related_videos : Bool
+ property speed : Float32 | Float64
+ property video_end : Float64 | Int32
+ property video_loop : Bool
+ property extend_desc : Bool
+ property video_start : Float64 | Int32
+ property volume : Int32
+ property vr_mode : Bool
+ property save_player_pos : Bool
+end
+
+def process_video_params(query, preferences)
+ annotations = query["iv_load_policy"]?.try &.to_i?
+ preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ comments = query["comments"]?.try &.split(",").map(&.downcase)
+ continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ player_style = query["player_style"]?
+ preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
+ quality = query["quality"]?
+ quality_dash = query["quality_dash"]?
+ region = query["region"]?
+ related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ speed = query["speed"]?.try &.rchop("x").to_f?
+ video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ volume = query["volume"]?.try &.to_i?
+ vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+
+ if preferences
+ # region ||= preferences.region
+ annotations ||= preferences.annotations.to_unsafe
+ preload ||= preferences.preload.to_unsafe
+ autoplay ||= preferences.autoplay.to_unsafe
+ comments ||= preferences.comments
+ continue ||= preferences.continue.to_unsafe
+ continue_autoplay ||= preferences.continue_autoplay.to_unsafe
+ listen ||= preferences.listen.to_unsafe
+ local ||= preferences.local.to_unsafe
+ player_style ||= preferences.player_style
+ preferred_captions ||= preferences.captions
+ quality ||= preferences.quality
+ quality_dash ||= preferences.quality_dash
+ related_videos ||= preferences.related_videos.to_unsafe
+ speed ||= preferences.speed
+ video_loop ||= preferences.video_loop.to_unsafe
+ extend_desc ||= preferences.extend_desc.to_unsafe
+ volume ||= preferences.volume
+ vr_mode ||= preferences.vr_mode.to_unsafe
+ save_player_pos ||= preferences.save_player_pos.to_unsafe
+ end
+
+ annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
+ preload ||= CONFIG.default_user_preferences.preload.to_unsafe
+ autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
+ comments ||= CONFIG.default_user_preferences.comments
+ continue ||= CONFIG.default_user_preferences.continue.to_unsafe
+ continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
+ listen ||= CONFIG.default_user_preferences.listen.to_unsafe
+ local ||= CONFIG.default_user_preferences.local.to_unsafe
+ player_style ||= CONFIG.default_user_preferences.player_style
+ preferred_captions ||= CONFIG.default_user_preferences.captions
+ quality ||= CONFIG.default_user_preferences.quality
+ quality_dash ||= CONFIG.default_user_preferences.quality_dash
+ related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
+ speed ||= CONFIG.default_user_preferences.speed
+ video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
+ extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
+ volume ||= CONFIG.default_user_preferences.volume
+ vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
+ save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
+
+ annotations = annotations == 1
+ preload = preload == 1
+ autoplay = autoplay == 1
+ continue = continue == 1
+ continue_autoplay = continue_autoplay == 1
+ listen = listen == 1
+ local = local == 1
+ related_videos = related_videos == 1
+ video_loop = video_loop == 1
+ extend_desc = extend_desc == 1
+ vr_mode = vr_mode == 1
+ save_player_pos = save_player_pos == 1
+
+ if CONFIG.disabled?("dash") && quality == "dash"
+ quality = "high"
+ end
+
+ if CONFIG.disabled?("local") && local
+ local = false
+ end
+
+ if start = query["t"]? || query["time_continue"]? || query["start"]?
+ video_start = decode_time(start)
+ end
+ video_start ||= 0
+
+ if query["end"]?
+ video_end = decode_time(query["end"])
+ end
+ video_end ||= -1
+
+ raw = query["raw"]?.try &.to_i?
+ raw ||= 0
+ raw = raw == 1
+
+ controls = query["controls"]?.try &.to_i?
+ controls ||= 1
+ controls = controls >= 1
+
+ params = VideoPreferences.new({
+ annotations: annotations,
+ preload: preload,
+ autoplay: autoplay,
+ comments: comments,
+ continue: continue,
+ continue_autoplay: continue_autoplay,
+ controls: controls,
+ listen: listen,
+ local: local,
+ player_style: player_style,
+ preferred_captions: preferred_captions,
+ quality: quality,
+ quality_dash: quality_dash,
+ raw: raw,
+ region: region,
+ related_videos: related_videos,
+ speed: speed,
+ video_end: video_end,
+ video_loop: video_loop,
+ extend_desc: extend_desc,
+ video_start: video_start,
+ volume: volume,
+ vr_mode: vr_mode,
+ save_player_pos: save_player_pos,
+ })
+
+ return params
+end
diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr
index ad50909a..6aea82ae 100644
--- a/src/invidious/views/add_playlist_items.ecr
+++ b/src/invidious/views/add_playlist_items.ecr
@@ -11,7 +11,9 @@
@@ -29,30 +31,5 @@
-
- <% videos.each_slice(4) do |slice| %>
- <% slice.each do |item| %>
- <%= rendered "components/item" %>
- <% end %>
- <% end %>
-
-<% if query %>
-
-
- <% if page > 1 %>
-
- <%= translate(locale, "Previous page") %>
-
- <% end %>
-
-
-
- <% if videos.size >= 20 %>
-
- <%= translate(locale, "Next page") %>
-
- <% end %>
-
-
-<% end %>
+<%= rendered "components/items_paginated" %>
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 40b553a9..686de6bd 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,116 +1,59 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url =
+ case selected_tab
+ when .shorts? then "/channel/#{ucid}/shorts"
+ when .streams? then "/channel/#{ucid}/streams"
+ when .playlists? then "/channel/#{ucid}/playlists"
+ when .channels? then "/channel/#{ucid}/channels"
+ when .podcasts? then "/channel/#{ucid}/podcasts"
+ when .releases? then "/channel/#{ucid}/releases"
+ when .courses? then "/channel/#{ucid}/courses"
+ else
+ "/channel/#{ucid}"
+ end
+
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
+ base_url: relative_url,
+ ctoken: next_continuation,
+ first_page: continuation.nil?,
+ params: env.params.query,
+ )
+%>
<% content_for "header" do %>
-<%= author %> - Invidious
+<%- if selected_tab.videos? -%>
+
+
+
+
+
+
+
+
+
+
+
+<%- end -%>
+
+
+
+
+<%= author %> - Invidious
<% end %>
-<% if channel.banner %>
-
- 
">
-
-
-
-
-
-<% end %>
-
-
-
-
-
- <%= channel.description_html %>
-
-
-
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
- <%= translate(locale, "View channel on YouTube") %>
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
- <% if !channel.auto_generated %>
-
- <%= translate(locale, "Videos") %>
-
- <% end %>
-
- <% if channel.auto_generated %>
- <%= translate(locale, "Playlists") %>
- <% else %>
- <%= translate(locale, "Playlists") %>
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
-
-
-
-
- <% sort_options.each do |sort| %>
-
- <% if sort_by == sort %>
- <%= translate(locale, sort) %>
- <% else %>
-
- <%= translate(locale, sort) %>
-
- <% end %>
-
- <% end %>
-
-
-
+<%= rendered "components/channel_info" %>
-
-<% items.each do |item| %>
- <%= rendered "components/item" %>
-<% end %>
-
-
-
- <% if page > 1 %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Previous page") %>
-
- <% end %>
-
-
-
- <% if count == 60 %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Next page") %>
-
- <% end %>
-
-
+<%= rendered "components/items_paginated" %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index f0add06b..132e636c 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,71 +1,21 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url = "/channel/#{ucid}/community"
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Posts
+-%>
<% content_for "header" do %>
+
<%= author %> - Invidious
<% end %>
-<% if channel.banner %>
-
- ">
-
-
-
-
-
-<% end %>
-
-
-
-
-
- <%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>
-
-
-
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
- <%= translate(locale, "View channel on YouTube") %>
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
- <% if !channel.auto_generated %>
-
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
-
-
-
+<%= rendered "components/channel_info" %>
@@ -76,8 +26,8 @@
<%= error_message %>
<% else %>
-
- <%= template_youtube_comments(items.not_nil!, locale, thin_mode) %>
+
+ <%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode) %>
<% end %>
diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr
new file mode 100644
index 00000000..f4164f31
--- /dev/null
+++ b/src/invidious/views/components/channel_info.ecr
@@ -0,0 +1,61 @@
+<% if channel.banner %>
+
+ " alt="" />
+
+
+
+
+
+<% end %>
+
+
+
+
+ 
+ <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %>
+
+
+
+
+
+
+
+ <%= channel.description_html %>
+
+
+
+
+
+
+
+ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
+
+
+
+ <% sort_options.each do |sort| %>
+
+ <% if sort_by == sort %>
+ <%= translate(locale, sort) %>
+ <% else %>
+ <%= translate(locale, sort) %>
+ <% end %>
+
+ <% end %>
+
+
+
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr
index 5f8bde13..a24423df 100644
--- a/src/invidious/views/components/item.ecr
+++ b/src/invidious/views/components/item.ecr
@@ -1,151 +1,199 @@
+<%-
+ thin_mode = env.get("preferences").as(Preferences).thin_mode
+ item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category | ProblematicTimelineItem) && env.get?("user").try &.as(User).watched.index(item.id) != nil
+ author_verified = item.responds_to?(:author_verified) && item.author_verified
+-%>
+
<% case item when %>
<% when SearchChannel %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
+ <% if !thin_mode %>
+
- .request_target.gsub(/=s\d+/, )
"/>
+ " alt="" />
- <% end %>
- <%= HTML.escape(item.author) %>
-
- <%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>
- <% if !item.auto_generated %><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
<% end %>
- <%= item.description_html %>
- <% when SearchPlaylist, InvidiousPlaylist %>
- <% if item.id.starts_with? "RD" %>
- <% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %>
- <% else %>
- <% url = "/playlist?list=#{item.id}" %>
+
+ <%- else -%>
+
<% end %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
- 
"/>
- <%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
-
- <% end %>
- <%= HTML.escape(item.title) %>
-
-
- <%= HTML.escape(item.author) %>
-
- <% when MixVideo %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
- 
- <% if item.length_seconds != 0 %>
- <%= recode_length_seconds(item.length_seconds) %>
- <% end %>
-
- <% end %>
- <%= HTML.escape(item.title) %>
-
-
- <%= HTML.escape(item.author) %>
-
- <% when PlaylistVideo %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
-
- <% if plid = env.get?("remove_playlist_items") %>
-
- <% end %>
-
- <% if item.responds_to?(:live_now) && item.live_now %>
- <%= translate(locale, "LIVE") %>
- <% elsif item.length_seconds != 0 %>
- <%= recode_length_seconds(item.length_seconds) %>
- <% end %>
-
- <% end %>
- <%= HTML.escape(item.title) %>
-
-
- <% endpoint_params = "?v=#{item.id}&list=#{item.plid}" %>
- <%= rendered "components/video-context-buttons" %>
+
+
+ <% if !item.channel_handle.nil? %><%= item.channel_handle %>
<% end %>
+ <%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>
+ <% if !item.auto_generated && item.channel_handle.nil? %><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
<% end %>
+ <%= item.description_html %>
+ <% when SearchHashtag %>
+ <% if !thin_mode %>
+
+ 
<%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
+ <%- end -%>
+
+
+
+ <%- if item.channel_count != 0 -%>
+ <%= translate_count(locale, "generic_channels_count", item.channel_count, NumberFormatting::Separator) %>
+ <%- end -%>
+
+ <% when SearchPlaylist, InvidiousPlaylist %>
+ <%-
+ if item.id.starts_with? "RD"
+ link_url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}"
+ else
+ link_url = "/playlist?list=#{item.id}"
+ end
+ -%>
+
+
+ <%- if !thin_mode %>
+
+ " alt="" />
+
+ <%- else -%>
+
+ <%- end -%>
+
+
+
+
+
- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
- <%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
- <% elsif Time.utc - item.published > 1.minute %>
- <%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
+ <% if !item.ucid.to_s.empty? %>
+
+ <%= HTML.escape(item.author) %>
+ <%- if author_verified %> <% end -%>
+
+
+ <% else %>
+ <%= HTML.escape(item.author) %>
+ <%- if author_verified %> <% end -%>
+
+ <% end %>
+
+
+ <% when Category %>
+ <% when ProblematicTimelineItem %>
+
+
+
+ <%=translate(locale, "timeline_parse_error_placeholder_heading")%>
+ <%=translate(locale, "timeline_parse_error_placeholder_message")%>
+
+
+ <%=translate(locale, "timeline_parse_error_show_technical_details")%>
+ <%=get_issue_template(env, item.parse_exception)[1]%>
+
+
+ <% else %>
+ <%-
+ # `endpoint_params` is used for the "video-context-buttons" component
+ if item.is_a?(PlaylistVideo)
+ link_url = "/watch?v=#{item.id}&list=#{item.plid}&index=#{item.index}"
+ endpoint_params = "?v=#{item.id}&list=#{item.plid}"
+ elsif item.is_a?(MixVideo)
+ link_url = "/watch?v=#{item.id}&list=#{item.rdid}"
+ endpoint_params = "?v=#{item.id}&list=#{item.rdid}"
+ else
+ link_url = "/watch?v=#{item.id}"
+ endpoint_params = "?v=#{item.id}"
+ end
+ -%>
+
+
+ <%- if !thin_mode -%>
+
+
+
+ <% if item_watched %>
+
+
+ <% end %>
+
+ <%- else -%>
+
+ <%- end -%>
+
+
+
+
+
+
+
+
+
+
+ <% if !item.ucid.to_s.empty? %>
+
+ <%= HTML.escape(item.author) %>
+ <%- if author_verified %> <% end -%>
+
+
+ <% else %>
+ <%= HTML.escape(item.author) %>
+ <%- if author_verified %> <% end -%>
+
<% end %>
- <% if item.responds_to?(:views) && item.views %>
-
- <%= translate_count(locale, "generic_views_count", item.views || 0, NumberFormatting::Short) %>
-
- <% end %>
-
- <% when Category %>
- <% else %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
-
- <% if env.get? "show_watched" %>
-
- <% elsif plid = env.get? "add_playlist_items" %>
-
- <% end %>
-
- <% if item.responds_to?(:live_now) && item.live_now %>
- <%= translate(locale, "LIVE") %>
- <% elsif item.length_seconds != 0 %>
- <%= recode_length_seconds(item.length_seconds) %>
- <% end %>
-
- <% end %>
- <%= HTML.escape(item.title) %>
-
-
-
-
-
- <% endpoint_params = "?v=#{item.id}" %>
<%= rendered "components/video-context-buttons" %>
@@ -153,7 +201,7 @@
<% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
- <% elsif Time.utc - item.published > 1.minute %>
+ <% elsif item.responds_to?(:published) && (Time.utc - item.published) > 1.minute %>
<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% end %>
diff --git a/src/invidious/views/components/items_paginated.ecr b/src/invidious/views/components/items_paginated.ecr
new file mode 100644
index 00000000..f69df3fe
--- /dev/null
+++ b/src/invidious/views/components/items_paginated.ecr
@@ -0,0 +1,21 @@
+<%= page_nav_html %>
+
+
+ <%- items.each do |item| -%>
+ <%= rendered "components/item" %>
+ <%- end -%>
+
+
+<%= page_nav_html %>
+
+
+
+
diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr
index 206ba380..af352102 100644
--- a/src/invidious/views/components/player.ecr
+++ b/src/invidious/views/components/player.ecr
@@ -1,18 +1,47 @@
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index aefa34cc..7c584d15 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -46,8 +46,14 @@ struct PlaylistVideo XML.build { |xml| to_xml(xml) } end + def to_json(locale : String?, json : JSON::Builder) + to_json(json) + end + def to_json(json : JSON::Builder, index : Int32? = nil) json.object do + json.field "type", "video" + json.field "title", self.title json.field "videoId", self.id @@ -56,7 +62,7 @@ struct PlaylistVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "videoThumbnails" do - generate_thumbnails(json, self.id) + Invidious::JSONify::APIv1.thumbnails(json, self.id) end if index @@ -67,6 +73,7 @@ struct PlaylistVideo end json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now end end @@ -89,6 +96,7 @@ struct Playlist property views : Int64 property updated : Time property thumbnail : String? + property subtitle : String? def to_json(offset, json : JSON::Builder, video_id : String? = nil) json.object do @@ -100,6 +108,7 @@ struct Playlist json.field "author", self.author json.field "authorId", self.ucid json.field "authorUrl", "/channel/#{self.ucid}" + json.field "subtitle", self.subtitle json.field "authorThumbnails" do json.array do @@ -261,7 +270,7 @@ end def subscribe_playlist(user, playlist) playlist = InvidiousPlaylist.new({ - title: playlist.title.byte_slice(0, 150), + title: playlist.title[..150], id: playlist.id, author: user.email, description: "", # Max 5000 characters @@ -317,7 +326,7 @@ def get_playlist(plid : String) if playlist = Invidious::Database::Playlists.select(id: plid) return playlist else - raise InfoException.new("Playlist does not exist.") + raise NotFoundException.new("Playlist does not exist.") end else return fetch_playlist(plid) @@ -356,12 +365,16 @@ def fetch_playlist(plid : String) updated = Time.utc video_count = 0 + subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle")) + playlist_info["stats"]?.try &.as_a.each do |stat| text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s next if !text if text.includes? "video" video_count = text.gsub(/\D/, "").to_i? || 0 + elsif text.includes? "episode" + video_count = text.gsub(/\D/, "").to_i? || 0 elsif text.includes? "view" views = text.gsub(/\D/, "").to_i64? || 0_i64 else @@ -397,6 +410,7 @@ def fetch_playlist(plid : String) views: views, updated: updated, thumbnail: thumbnail, + subtitle: subtitle, }) end @@ -418,7 +432,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset end - videos = [] of PlaylistVideo + videos = [] of PlaylistVideo | ProblematicTimelineItem until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count # 100 videos per request @@ -434,7 +448,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) - videos = [] of PlaylistVideo + videos = [] of PlaylistVideo | ProblematicTimelineItem if initial_data["contents"]? tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] @@ -486,12 +500,14 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) index: index, }) end + rescue ex + videos << ProblematicTimelineItem.new(parse_exception: ex) end return videos end -def template_playlist(playlist) +def template_playlist(playlist, listen) html = <<-END_HTML
@@ -505,9 +521,9 @@ def template_playlist(playlist)
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr
index 9bb73136..c8db207c 100644
--- a/src/invidious/routes/account.cr
+++ b/src/invidious/routes/account.cr
@@ -42,11 +42,6 @@ module Invidious::Routes::Account
sid = sid.as(String)
token = env.params.body["csrf_token"]?
- # We don't store passwords for Google accounts
- if !user.password
- return error_template(400, "Cannot change password for Google accounts")
- end
-
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
@@ -54,11 +49,11 @@ module Invidious::Routes::Account
end
password = env.params.body["password"]?
- if !password
+ if password.nil? || password.empty?
return error_template(401, "Password is a required field")
end
- new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v }
+ new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match")
@@ -203,7 +198,7 @@ module Invidious::Routes::Account
referer = get_referer(env)
if !user
- return env.redirect referer
+ return env.redirect "/login?referer=#{URI.encode_path_segment(env.request.resource)}"
end
user = user.as(User)
@@ -245,7 +240,7 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v }
+ scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
@@ -262,6 +257,7 @@ module Invidious::Routes::Account
end
query["token"] = access_token
+ query["username"] = URI.encode_path_segment(user.email)
url.query = query.to_s
env.redirect url.to_s
@@ -332,17 +328,9 @@ module Invidious::Routes::Account
end
end
- if env.params.query["action_revoke_token"]?
- action = "action_revoke_token"
- else
- return env.redirect referer
- end
-
- session = env.params.query["session"]?
- session ||= ""
-
- case action
- when .starts_with? "action_revoke_token"
+ case action = env.params.query["action"]?
+ when "revoke_token"
+ session = env.params.query["session"]
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
return error_json(400, "Unsupported action #{action}")
diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr
index ca429df5..c27caad7 100644
--- a/src/invidious/routes/api/manifest.cr
+++ b/src/invidious/routes/api/manifest.cr
@@ -8,45 +8,49 @@ module Invidious::Routes::API::Manifest
id = env.params.url["id"]
region = env.params.query["region"]?
+ if CONFIG.invidious_companion.present?
+ invidious_companion = CONFIG.invidious_companion.sample
+ return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
+ end
+
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin
video = get_video(id, region: region)
- rescue ex : VideoRedirect
- return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex : NotFoundException
+ haltf env, status_code: 404
rescue ex
haltf env, status_code: 403
end
if dashmpd = video.dash_manifest_url
- manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body
+ response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
- manifest = manifest.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
- url = baseurl.lchop("")
- url = url.rchop(" ")
-
- if local
- uri = URI.parse(url)
- url = "#{uri.request_target}host/#{uri.host}/"
- end
+ if response.status_code != 200
+ haltf env, status_code: response.status_code
+ end
+ # Proxy URLs for video playback on invidious.
+ # Other API clients can get the original URLs by omiting `local=true`.
+ manifest = response.body.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
+ url = baseurl.lchop("").rchop(" ")
+ url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
"#{url} "
end
return manifest
end
- adaptive_fmts = video.adaptive_fmts
-
+ # Ditto, only proxify URLs if `local=true` is used
if local
- adaptive_fmts.each do |fmt|
- fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target)
+ video.adaptive_fmts.each do |fmt|
+ fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
end
end
- audio_streams = video.audio_streams
+ audio_streams = video.audio_streams.sort_by { |stream| {stream["bitrate"].as_i} }.reverse!
video_streams = video.video_streams.sort_by { |stream| {stream["width"].as_i, stream["fps"].as_i} }.reverse!
manifest = XML.build(indent: " ", encoding: "UTF-8") do |xml|
@@ -56,17 +60,32 @@ module Invidious::Routes::API::Manifest
xml.element("Period") do
i = 0
- {"audio/mp4", "audio/webm"}.each do |mime_type|
+ {"audio/mp4"}.each do |mime_type|
mime_streams = audio_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
- xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true) do
- mime_streams.each do |fmt|
+ mime_streams.each do |fmt|
+ # OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
+ next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
+
+ audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any
+ lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und"
+ is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0
+ displayname = audio_track["displayName"]?.try &.as_s || "Unknown"
+ bitrate = fmt["bitrate"]
+
+ # Different representations of the same audio should be groupped into one AdaptationSet.
+ # However, most players don't support auto quality switching, so we have to trick them
+ # into providing a quality selector.
+ # See https://github.com/iv-org/invidious/issues/3074 for more details.
+ xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
+ xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate")
+
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
value: "2")
@@ -76,20 +95,22 @@ module Invidious::Routes::API::Manifest
end
end
end
+ i += 1
end
-
- i += 1
end
potential_heights = {4320, 2160, 1440, 1080, 720, 480, 360, 240, 144}
- {"video/mp4", "video/webm"}.each do |mime_type|
+ {"video/mp4"}.each do |mime_type|
mime_streams = video_streams.select { |stream| stream["mimeType"].as_s.starts_with? mime_type }
next if mime_streams.empty?
heights = [] of Int32
xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, scanType: "progressive") do
mime_streams.each do |fmt|
+ # OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
+ next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
+
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
@@ -160,8 +181,9 @@ module Invidious::Routes::API::Manifest
manifest = response.body
if local
- manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
- path = URI.parse(match).path
+ manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match|
+ uri = URI.parse(match)
+ path = uri.path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
@@ -190,7 +212,7 @@ module Invidious::Routes::API::Manifest
raw_params["fvip"] = fvip["fvip"]
end
- raw_params["local"] = "true"
+ raw_params["host"] = uri.host.not_nil!
"#{HOST_URL}/videoplayback?#{raw_params}"
end
diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr
index b559a01a..a35d2f2b 100644
--- a/src/invidious/routes/api/v1/authenticated.cr
+++ b/src/invidious/routes/api/v1/authenticated.cr
@@ -31,6 +31,88 @@ module Invidious::Routes::API::V1::Authenticated
env.response.status_code = 204
end
+ def self.export_invidious(env)
+ env.response.content_type = "application/json"
+ user = env.get("user").as(User)
+
+ return Invidious::User::Export.to_invidious(user)
+ end
+
+ def self.import_invidious(env)
+ user = env.get("user").as(User)
+
+ begin
+ if body = env.request.body
+ body = env.request.body.not_nil!.gets_to_end
+ else
+ body = "{}"
+ end
+ Invidious::User::Import.from_invidious(user, body)
+ rescue
+ end
+
+ env.response.status_code = 204
+ end
+
+ def self.get_history(env)
+ env.response.content_type = "application/json"
+ user = env.get("user").as(User)
+
+ page = env.params.query["page"]?.try &.to_i?.try &.clamp(0, Int32::MAX)
+ page ||= 1
+
+ max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
+ max_results ||= user.preferences.max_results
+ max_results ||= CONFIG.default_user_preferences.max_results
+
+ start_index = (page - 1) * max_results
+ if user.watched[start_index]?
+ watched = user.watched.reverse[start_index, max_results]
+ end
+ watched ||= [] of String
+
+ return watched.to_json
+ end
+
+ def self.mark_watched(env)
+ user = env.get("user").as(User)
+
+ if !user.preferences.watch_history
+ return error_json(409, "Watch history is disabled in preferences.")
+ end
+
+ id = env.params.url["id"]
+ if !id.match(/^[a-zA-Z0-9_-]{11}$/)
+ return error_json(400, "Invalid video id.")
+ end
+
+ Invidious::Database::Users.mark_watched(user, id)
+ env.response.status_code = 204
+ end
+
+ def self.mark_unwatched(env)
+ user = env.get("user").as(User)
+
+ if !user.preferences.watch_history
+ return error_json(409, "Watch history is disabled in preferences.")
+ end
+
+ id = env.params.url["id"]
+ if !id.match(/^[a-zA-Z0-9_-]{11}$/)
+ return error_json(400, "Invalid video id.")
+ end
+
+ Invidious::Database::Users.mark_unwatched(user, id)
+ env.response.status_code = 204
+ end
+
+ def self.clear_history(env)
+ user = env.get("user").as(User)
+
+ Invidious::Database::Users.clear_watch_history(user)
+ env.response.status_code = 204
+ end
+
def self.feed(env)
env.response.content_type = "application/json"
@@ -96,10 +178,6 @@ module Invidious::Routes::API::V1::Authenticated
Invidious::Database::Users.subscribe_channel(user, ucid)
end
- # For Google accounts, access tokens don't have enough information to
- # make a request on the user's behalf, which is why we don't sync with
- # YouTube.
-
env.response.status_code = 204
end
@@ -226,8 +304,8 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(403, "Invalid user")
end
- if playlist.index.size >= 500
- return error_json(400, "Playlist cannot have more than 500 videos")
+ if playlist.index.size >= CONFIG.playlist_length_limit
+ return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
video_id = env.params.json["videoId"].try &.as(String)
@@ -237,6 +315,8 @@ module Invidious::Routes::API::V1::Authenticated
begin
video = get_video(video_id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index c4d6643a..a940ee68 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -1,29 +1,44 @@
module Invidious::Routes::API::V1::Channels
- def self.home(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "newest"
-
+ # Macro to avoid duplicating some code below
+ # This sets the `channel` variable, or handles Exceptions.
+ private macro get_channel
begin
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
+ end
- page = 1
- if channel.auto_generated
- videos = [] of SearchVideo
- count = 0
+ def self.home(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve "sort by" setting from URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+
+ if channel.is_age_gated
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
+ videos = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ videos = [] of PlaylistVideo
+ end
+ next_continuation = nil
else
begin
- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
rescue ex
return error_json(500, ex)
end
@@ -80,11 +95,15 @@ module Invidious::Routes::API::V1::Channels
json.field "joined", channel.joined.to_unix
json.field "autoGenerated", channel.auto_generated
+ json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html
json.field "allowedRegions", channel.allowed_regions
+ json.field "tabs", channel.tabs
+ json.field "tags", channel.tags
+ json.field "authorVerified", channel.verified
json.field "latestVideos" do
json.array do
@@ -98,31 +117,13 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
- related_channels = fetch_related_channels(channel)
+ related_channels, _ = fetch_related_channels(channel)
rescue ex
- related_channels = [] of AboutRelatedChannel
+ related_channels = [] of SearchChannel
end
related_channels.each do |related_channel|
- json.object do
- json.field "author", related_channel.author
- json.field "authorId", related_channel.ucid
- json.field "authorUrl", related_channel.author_url
-
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
- end
+ related_channel.to_json(locale, json)
end
end
end # relatedChannels
@@ -132,59 +133,147 @@ module Invidious::Routes::API::V1::Channels
end
def self.latest(env)
- locale = env.get("preferences").as(Preferences).locale
+ # Remove parameters that could affect this endpoint's behavior
+ env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
+ env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
-
- begin
- videos = get_latest_videos(ucid)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.array do
- videos.each do |video|
- video.to_json(locale, json)
- end
- end
- end
+ return self.videos(env)
end
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
env.response.content_type = "application/json"
- ucid = env.params.url["ucid"]
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
- sort_by = env.params.query["sort"]?.try &.downcase
- sort_by ||= env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "newest"
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
- rescue ex
- return error_json(500, ex)
+ # Retrieve some URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ continuation = env.params.query["continuation"]?
+
+ if channel.is_age_gated
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
+ videos = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ videos = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
end
- begin
- count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
- rescue ex
- return error_json(500, ex)
- end
-
- JSON.build do |json|
- json.array do
- videos.each do |video|
- video.to_json(locale, json)
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.shorts(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve continuation from URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ continuation = env.params.query["continuation"]?
+
+ if channel.is_age_gated
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
+ videos = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ videos = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ begin
+ videos, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.streams(env)
+ locale = env.get("preferences").as(Preferences).locale
+ ucid = env.params.url["ucid"]
+
+ env.response.content_type = "application/json"
+
+ # Use the private macro defined above.
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ # Retrieve continuation from URL parameters
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ continuation = env.params.query["continuation"]?
+
+ if channel.is_age_gated
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
+ videos = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ videos = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ rescue ex
+ return error_json(500, ex)
+ end
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "videos" do
+ json.array do
+ videos.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
end
end
end
@@ -200,16 +289,11 @@ module Invidious::Routes::API::V1::Channels
env.params.query["sort_by"]?.try &.downcase ||
"last"
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
- return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
- rescue ex
- return error_json(500, ex)
- end
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+ items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
JSON.build do |json|
json.object do
@@ -221,7 +305,94 @@ module Invidious::Routes::API::V1::Channels
end
end
- json.field "continuation", continuation
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.podcasts(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ continuation = env.params.query["continuation"]?
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ items, next_continuation = fetch_channel_podcasts(channel.ucid, channel.author, continuation)
+
+ JSON.build do |json|
+ json.object do
+ json.field "playlists" do
+ json.array do
+ items.each do |item|
+ item.to_json(locale, json) if item.is_a?(SearchPlaylist)
+ end
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.releases(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ continuation = env.params.query["continuation"]?
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ items, next_continuation = fetch_channel_releases(channel.ucid, channel.author, continuation)
+
+ JSON.build do |json|
+ json.object do
+ json.field "playlists" do
+ json.array do
+ items.each do |item|
+ item.to_json(locale, json) if item.is_a?(SearchPlaylist)
+ end
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.courses(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ ucid = env.params.url["ucid"]
+ continuation = env.params.query["continuation"]?
+
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation)
+
+ JSON.build do |json|
+ json.object do
+ json.field "playlists" do
+ json.array do
+ items.each do |item|
+ item.to_json(locale, json) if item.is_a?(SearchPlaylist)
+ end
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
end
end
end
@@ -249,20 +420,108 @@ module Invidious::Routes::API::V1::Channels
end
end
- def self.search(env)
+ def self.post(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+ id = env.params.url["id"].to_s
+ ucid = env.params.query["ucid"]?
+
+ thin_mode = env.params.query["thin_mode"]?
+ thin_mode = thin_mode == "true"
+
+ format = env.params.query["format"]?
+ format ||= "json"
+
+ if ucid.nil?
+ response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
+ return error_json(400, "Invalid post ID") if response["error"]?
+ ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+ else
+ ucid = ucid.to_s
+ end
+
+ begin
+ fetch_channel_community_post(ucid, id, locale, format, thin_mode)
+ rescue ex
+ return error_json(500, ex)
+ end
+ end
+
+ def self.post_comments(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
+ id = env.params.url["id"]
+
+ thin_mode = env.params.query["thin_mode"]?
+ thin_mode = thin_mode == "true"
+
+ format = env.params.query["format"]?
+ format ||= "json"
+
+ continuation = env.params.query["continuation"]?
+
+ case continuation
+ when nil, ""
+ ucid = env.params.query["ucid"]
+ comments = Comments.fetch_community_post_comments(ucid, id)
+ else
+ comments = YoutubeAPI.browse(continuation: continuation)
+ end
+ return Comments.parse_youtube(id, comments, format, locale, thin_mode, is_post: true)
+ end
+
+ def self.channels(env)
+ locale = env.get("preferences").as(Preferences).locale
ucid = env.params.url["ucid"]
- query = env.params.query["q"]?
- query ||= ""
+ env.response.content_type = "application/json"
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
+ # Use the macro defined above
+ channel = nil # Make the compiler happy
+ get_channel()
+
+ continuation = env.params.query["continuation"]?
+
+ begin
+ items, next_continuation = fetch_related_channels(channel, continuation)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ JSON.build do |json|
+ json.object do
+ json.field "relatedChannels" do
+ json.array do
+ items.each &.to_json(locale, json)
+ end
+ end
+
+ json.field "continuation", next_continuation if next_continuation
+ end
+ end
+ end
+
+ def self.search(env)
+ locale = env.get("preferences").as(Preferences).locale
+ region = env.params.query["region"]?
+
+ env.response.content_type = "application/json"
+
+ query = Invidious::Search::Query.new(env.params.query, :channel, region)
+
+ # Required because we can't (yet) pass multiple parameter to the
+ # `Search::Query` initializer (in this case, an URL segment)
+ query.channel = env.params.url["ucid"]
+
+ begin
+ search_results = query.process
+ rescue ex
+ return error_json(400, ex)
+ end
- search_results = channel_search(query, page, ucid)
JSON.build do |json|
json.array do
search_results.each do |item|
diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr
index 41865f34..fea2993c 100644
--- a/src/invidious/routes/api/v1/feeds.cr
+++ b/src/invidious/routes/api/v1/feeds.cr
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds
if !CONFIG.popular_enabled
error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
- haltf env, 400, error_message
+ haltf env, 403, error_message
end
JSON.build do |json|
diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr
index 844fedb8..4f5b58da 100644
--- a/src/invidious/routes/api/v1/misc.cr
+++ b/src/invidious/routes/api/v1/misc.cr
@@ -6,6 +6,22 @@ module Invidious::Routes::API::V1::Misc
if !CONFIG.statistics_enabled
return {"software" => SOFTWARE}.to_json
else
+ # Calculate playback success rate
+ if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]?)
+ tracker = tracker.as(Hash(String, Int64 | Float64))
+
+ if !tracker.empty?
+ total_requests = tracker["totalRequests"]
+ success_count = tracker["successfulRequests"]
+
+ if total_requests.zero?
+ tracker["ratio"] = 1_i64
+ else
+ tracker["ratio"] = (success_count / (total_requests)).round(2)
+ end
+ end
+ end
+
return Invidious::Jobs::StatisticsRefreshJob::STATISTICS.to_json
end
end
@@ -26,6 +42,9 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
+ listen_param = env.params.query["listen"]?
+ listen = (listen_param == "true" || listen_param == "1")
+
if plid.starts_with? "RD"
return env.redirect "/api/v1/mixes/#{plid}"
end
@@ -58,7 +77,9 @@ module Invidious::Routes::API::V1::Misc
response = playlist.to_json(offset, video_id: video_id)
json_response = JSON.parse(response)
- if json_response["videos"].as_a[0]["index"] != offset
+ if json_response["videos"].as_a.empty?
+ json_response = JSON.parse(response)
+ elsif json_response["videos"].as_a[0]["index"] != offset
offset = json_response["videos"].as_a[0]["index"].as_i
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback)
@@ -67,7 +88,7 @@ module Invidious::Routes::API::V1::Misc
end
if format == "html"
- playlist_html = template_playlist(json_response)
+ playlist_html = template_playlist(json_response, listen)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = {
@@ -93,6 +114,9 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
+ listen_param = env.params.query["listen"]?
+ listen = (listen_param == "true" || listen_param == "1")
+
begin
mix = fetch_mix(rdid, continuation, locale: locale)
@@ -123,9 +147,7 @@ module Invidious::Routes::API::V1::Misc
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "videoThumbnails" do
- json.array do
- generate_thumbnails(json, video.id)
- end
+ Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
json.field "index", video.index
@@ -139,7 +161,7 @@ module Invidious::Routes::API::V1::Misc
if format == "html"
response = JSON.parse(response)
- playlist_html = template_mix(response)
+ playlist_html = template_mix(response, listen)
next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"]
response = {
@@ -150,4 +172,36 @@ module Invidious::Routes::API::V1::Misc
response
end
+
+ # resolve channel and clip urls, return the UCID
+ def self.resolve_url(env)
+ env.response.content_type = "application/json"
+ url = env.params.query["url"]?
+
+ return error_json(400, "Missing URL to resolve") if !url
+
+ begin
+ resolved_url = YoutubeAPI.resolve_url(url.as(String))
+ endpoint = resolved_url["endpoint"]
+ page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
+ if page_type == "WEB_PAGE_TYPE_UNKNOWN"
+ return error_json(400, "Unknown url")
+ end
+
+ sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
+ params = sub_endpoint.try &.dig?("params")
+ rescue ex
+ return error_json(500, ex)
+ end
+ JSON.build do |json|
+ json.object do
+ json.field "ucid", sub_endpoint["browseId"].as_s if sub_endpoint["browseId"]?
+ json.field "videoId", sub_endpoint["videoId"].as_s if sub_endpoint["videoId"]?
+ json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
+ json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
+ json.field "params", params.try &.as_s
+ json.field "pageType", page_type
+ end
+ end
+ end
end
diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 0b0853b1..59a30745 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -5,34 +5,14 @@ module Invidious::Routes::API::V1::Search
env.response.content_type = "application/json"
- query = env.params.query["q"]?
- query ||= ""
-
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
-
- sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "relevance"
-
- date = env.params.query["date"]?.try &.downcase
- date ||= ""
-
- duration = env.params.query["duration"]?.try &.downcase
- duration ||= ""
-
- features = env.params.query["features"]?.try &.split(",").map(&.downcase)
- features ||= [] of String
-
- content_type = env.params.query["type"]?.try &.downcase
- content_type ||= "video"
+ query = Invidious::Search::Query.new(env.params.query, :regular, region)
begin
- search_params = produce_search_params(page, sort_by, date, content_type, duration, features)
+ search_results = query.process
rescue ex
return error_json(400, ex)
end
- search_results = search(query, search_params, region)
JSON.build do |json|
json.array do
search_results.each do |item|
@@ -43,20 +23,21 @@ module Invidious::Routes::API::V1::Search
end
def self.search_suggestions(env)
- locale = env.get("preferences").as(Preferences).locale
- region = env.params.query["region"]?
+ preferences = env.get("preferences").as(Preferences)
+ region = env.params.query["region"]? || preferences.region
env.response.content_type = "application/json"
- query = env.params.query["q"]?
- query ||= ""
+ query = env.params.query["q"]? || ""
begin
- headers = HTTP::Headers{":authority" => "suggestqueries.google.com"}
- response = YT_POOL.client &.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback", headers).body
+ client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
+ url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
- body = response[35..-2]
- body = JSON.parse(body).as_a
+ response = client.get(url).body
+ client.close
+
+ body = JSON.parse(response[19..-2]).as_a
suggestions = body[1].as_a[0..-2]
JSON.build do |json|
@@ -75,4 +56,32 @@ module Invidious::Routes::API::V1::Search
return error_json(500, ex)
end
end
+
+ def self.hashtag(env)
+ hashtag = env.params.url["hashtag"]
+
+ page = env.params.query["page"]?.try &.to_i? || 1
+
+ locale = env.get("preferences").as(Preferences).locale
+ region = env.params.query["region"]?
+ env.response.content_type = "application/json"
+
+ begin
+ results = Invidious::Hashtag.fetch(hashtag, page, region)
+ rescue ex
+ return error_json(400, ex)
+ end
+
+ JSON.build do |json|
+ json.object do
+ json.field "results" do
+ json.array do
+ results.each do |item|
+ item.to_json(locale, json)
+ end
+ end
+ end
+ end
+ end
+ end
end
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index a9f891f5..6a3eb8ae 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -1,3 +1,5 @@
+require "html"
+
module Invidious::Routes::API::V1::Videos
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
@@ -6,17 +8,19 @@ module Invidious::Routes::API::V1::Videos
id = env.params.url["id"]
region = env.params.query["region"]?
+ proxy = {"1", "true"}.any? &.== env.params.query["local"]?
begin
video = get_video(id, region: region)
- rescue ex : VideoRedirect
- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
- video.to_json(locale, nil)
+ return JSON.build do |json|
+ Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
+ end
end
def self.captions(env)
@@ -39,9 +43,8 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
- rescue ex : VideoRedirect
- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
+ rescue ex : NotFoundException
+ haltf env, 404
rescue ex
haltf env, 500
end
@@ -86,61 +89,77 @@ module Invidious::Routes::API::V1::Videos
caption = caption[0]
end
- url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
+ if CONFIG.use_innertube_for_captions
+ params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
- # Auto-generated captions often have cues that aren't aligned properly with the video,
- # as well as some other markup that makes it cumbersome, so we try to fix that here
- if caption.name.includes? "auto-generated"
- caption_xml = YT_POOL.client &.get(url).body
- caption_xml = XML.parse(caption_xml)
+ transcript = Invidious::Videos::Transcript.from_raw(
+ YoutubeAPI.get_transcript(params),
+ caption.language_code,
+ caption.auto_generated
+ )
- webvtt = String.build do |str|
- str << <<-END_VTT
- WEBVTT
- Kind: captions
- Language: #{tlang || caption.language_code}
+ webvtt = transcript.to_vtt
+ else
+ # Timedtext API handling
+ url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
+ # Auto-generated captions often have cues that aren't aligned properly with the video,
+ # as well as some other markup that makes it cumbersome, so we try to fix that here
+ if caption.name.includes? "auto-generated"
+ caption_xml = YT_POOL.client &.get(url).body
- END_VTT
+ settings_field = {
+ "Kind" => "captions",
+ "Language" => "#{tlang || caption.language_code}",
+ }
- caption_nodes = caption_xml.xpath_nodes("//transcript/text")
- caption_nodes.each_with_index do |node, i|
- start_time = node["start"].to_f.seconds
- duration = node["dur"]?.try &.to_f.seconds
- duration ||= start_time
+ if caption_xml.starts_with?(" i + 1
- end_time = caption_nodes[i + 1]["start"].to_f.seconds
- else
- end_time = start_time + duration
+ webvtt = WebVTT.build(settings_field) do |builder|
+ caption_nodes = caption_xml.xpath_nodes("//transcript/text")
+ caption_nodes.each_with_index do |node, i|
+ start_time = node["start"].to_f.seconds
+ duration = node["dur"]?.try &.to_f.seconds
+ duration ||= start_time
+
+ if caption_nodes.size > i + 1
+ end_time = caption_nodes[i + 1]["start"].to_f.seconds
+ else
+ end_time = start_time + duration
+ end
+
+ text = HTML.unescape(node.content)
+ text = text.gsub(//, "")
+ text = text.gsub(/<\/font>/, "")
+ if md = text.match(/(?.*) : (?.*)/)
+ text = "#{md["text"]} "
+ end
+
+ builder.cue(start_time, end_time, text)
+ end
end
+ end
+ else
+ uri = URI.parse(url)
+ query_params = uri.query_params
+ query_params["fmt"] = "vtt"
+ uri.query_params = query_params
+ webvtt = YT_POOL.client &.get(uri.request_target).body
- start_time = "#{start_time.hours.to_s.rjust(2, '0')}:#{start_time.minutes.to_s.rjust(2, '0')}:#{start_time.seconds.to_s.rjust(2, '0')}.#{start_time.milliseconds.to_s.rjust(3, '0')}"
- end_time = "#{end_time.hours.to_s.rjust(2, '0')}:#{end_time.minutes.to_s.rjust(2, '0')}:#{end_time.seconds.to_s.rjust(2, '0')}.#{end_time.milliseconds.to_s.rjust(3, '0')}"
-
- text = HTML.unescape(node.content)
- text = text.gsub(//, "")
- text = text.gsub(/<\/font>/, "")
- if md = text.match(/(?.*) : (?.*)/)
- text = "#{md["text"]} "
- end
-
- str << <<-END_CUE
- #{start_time} --> #{end_time}
- #{text}
-
-
- END_CUE
+ if webvtt.starts_with?(" [0-9:.]{12}).+/, "\\1")
end
end
- else
- # Some captions have "align:[start/end]" and "position:[num]%"
- # attributes. Those are causing issues with VideoJS, which is unable
- # to properly align the captions on the video, so we remove them.
- #
- # See: https://github.com/iv-org/invidious/issues/2391
- webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
- .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
end
if title = env.params.query["title"]?
@@ -164,22 +183,20 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
- rescue ex : VideoRedirect
- env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
- return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
+ rescue ex : NotFoundException
+ haltf env, 404
rescue ex
haltf env, 500
end
- storyboards = video.storyboards
- width = env.params.query["width"]?
- height = env.params.query["height"]?
+ width = env.params.query["width"]?.try &.to_i
+ height = env.params.query["height"]?.try &.to_i
if !width && !height
response = JSON.build do |json|
json.object do
json.field "storyboards" do
- generate_storyboards(json, id, storyboards)
+ Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards)
end
end
end
@@ -189,43 +206,48 @@ module Invidious::Routes::API::V1::Videos
env.response.content_type = "text/vtt"
- storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" }
+ # Select a storyboard matching the user's provided width/height
+ storyboard = video.storyboards.select { |x| x.width == width || x.height == height }
+ haltf env, 404 if storyboard.empty?
- if storyboard.empty?
- haltf env, 404
- else
- storyboard = storyboard[0]
- end
+ # Alias variable, to make the code below esaier to read
+ sb = storyboard[0]
- String.build do |str|
- str << <<-END_VTT
- WEBVTT
- END_VTT
+ # Some base URL segments that we'll use to craft the final URLs
+ work_url = sb.proxied_url.dup
+ template_path = sb.proxied_url.path
- start_time = 0.milliseconds
- end_time = storyboard[:interval].milliseconds
+ # Initialize cue timing variables
+ # NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap
+ # (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000)
+ time_delta = sb.interval.milliseconds
+ start_time = 0.milliseconds
+ end_time = time_delta
- storyboard[:storyboard_count].times do |i|
- url = storyboard[:url]
- authority = /(i\d?).ytimg.com/.match(url).not_nil![1]?
- url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "")
- url = "#{HOST_URL}/sb/#{authority}/#{url}"
+ # Build a VTT file for VideoJS-vtt plugin
+ vtt_file = WebVTT.build do |vtt|
+ sb.images_count.times do |i|
+ # Replace the variable component part of the path
+ work_url.path = template_path.sub("$M", i)
- storyboard[:storyboard_height].times do |j|
- storyboard[:storyboard_width].times do |k|
- str << <<-END_CUE
- #{start_time}.000 --> #{end_time}.000
- #{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}
+ sb.rows.times do |j|
+ sb.columns.times do |k|
+ # The URL fragment represents the offset of the thumbnail inside the storyboard image
+ work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}"
+ vtt.cue(start_time, end_time, work_url.to_s)
- END_CUE
-
- start_time += storyboard[:interval].milliseconds
- end_time += storyboard[:interval].milliseconds
+ start_time += time_delta
+ end_time += time_delta
end
end
end
end
+
+ # videojs-vtt-thumbnails is not compliant to the VTT specification, it
+ # doesn't unescape the HTML entities, so we have to do it here:
+ # TODO: remove this when we migrate to VideoJS 8
+ return HTML.unescape(vtt_file)
end
def self.annotations(env)
@@ -246,7 +268,7 @@ module Invidious::Routes::API::V1::Videos
if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
annotations = cached_annotation.annotations
else
- index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0')
+ index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
# IA doesn't handle leading hyphens,
# so we use https://archive.org/details/youtubeannotations_64
@@ -323,7 +345,9 @@ module Invidious::Routes::API::V1::Videos
sort_by ||= "top"
begin
- comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
+ comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
return error_json(500, ex)
end
@@ -333,7 +357,7 @@ module Invidious::Routes::API::V1::Videos
sort_by ||= "confidence"
begin
- comments, reddit_thread = fetch_reddit_comments(id, sort_by: sort_by)
+ comments, reddit_thread = Comments.fetch_reddit(id, sort_by: sort_by)
rescue ex
comments = nil
reddit_thread = nil
@@ -349,9 +373,9 @@ module Invidious::Routes::API::V1::Videos
return reddit_thread.to_json
else
- content_html = template_reddit_comments(comments, locale)
- content_html = fill_links(content_html, "https", "www.reddit.com")
- content_html = replace_links(content_html)
+ content_html = Frontend::Comments.template_reddit(comments, locale)
+ content_html = Comments.fill_links(content_html, "https", "www.reddit.com")
+ content_html = Comments.replace_links(content_html)
response = {
"title" => reddit_thread.title,
"permalink" => reddit_thread.permalink,
@@ -362,4 +386,133 @@ module Invidious::Routes::API::V1::Videos
end
end
end
+
+ def self.clips(env)
+ locale = env.get("preferences").as(Preferences).locale
+
+ env.response.content_type = "application/json"
+
+ clip_id = env.params.url["id"]
+ region = env.params.query["region"]?
+ proxy = {"1", "true"}.any? &.== env.params.query["local"]?
+
+ response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
+ return error_json(400, "Invalid clip ID") if response["error"]?
+
+ video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
+ return error_json(400, "Invalid clip ID") if video_id.nil?
+
+ start_time = nil
+ end_time = nil
+ clip_title = nil
+
+ if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
+ start_time, end_time, clip_title = parse_clip_parameters(params)
+ end
+
+ begin
+ video = get_video(video_id, region: region)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return JSON.build do |json|
+ json.object do
+ json.field "startTime", start_time
+ json.field "endTime", end_time
+ json.field "clipTitle", clip_title
+ json.field "video" do
+ Invidious::JSONify::APIv1.video(video, json, locale: locale, proxy: proxy)
+ end
+ end
+ end
+ end
+
+ # Fetches transcripts from YouTube
+ #
+ # Use the `lang` and `autogen` query parameter to select which transcript to fetch
+ # Request without any URL parameters to see all the available transcripts.
+ def self.transcripts(env)
+ env.response.content_type = "application/json"
+
+ id = env.params.url["id"]
+ lang = env.params.query["lang"]?
+ label = env.params.query["label"]?
+ auto_generated = env.params.query["autogen"]? ? true : false
+
+ # Return all available transcript options when none is given
+ if !label && !lang
+ begin
+ video = get_video(id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ response = JSON.build do |json|
+ # The amount of transcripts available to fetch is the
+ # same as the amount of captions available.
+ available_transcripts = video.captions
+
+ json.object do
+ json.field "transcripts" do
+ json.array do
+ available_transcripts.each do |transcript|
+ json.object do
+ json.field "label", transcript.name
+ json.field "languageCode", transcript.language_code
+ json.field "autoGenerated", transcript.auto_generated
+
+ if transcript.auto_generated
+ json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}&autogen"
+ else
+ json.field "url", "/api/v1/transcripts/#{id}?lang=#{URI.encode_www_form(transcript.language_code)}"
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+
+ return response
+ end
+
+ # If lang is not given then we attempt to fetch
+ # the transcript through the given label
+ if lang.nil?
+ begin
+ video = get_video(id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ target_transcript = video.captions.select(&.name.== label)
+ if target_transcript.empty?
+ return error_json(404, NotFoundException.new("Requested transcript does not exist"))
+ else
+ target_transcript = target_transcript[0]
+ lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated
+ end
+ end
+
+ params = Invidious::Videos::Transcript.generate_param(id, lang, auto_generated)
+
+ begin
+ transcript = Invidious::Videos::Transcript.from_raw(
+ YoutubeAPI.get_transcript(params), lang, auto_generated
+ )
+ rescue ex : NotFoundException
+ return error_json(404, ex)
+ rescue ex
+ return error_json(500, ex)
+ end
+
+ return transcript.to_json
+ end
end
diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr
new file mode 100644
index 00000000..b5269668
--- /dev/null
+++ b/src/invidious/routes/before_all.cr
@@ -0,0 +1,133 @@
+module Invidious::Routes::BeforeAll
+ def self.handle(env)
+ preferences = Preferences.from_json("{}")
+
+ begin
+ if prefs_cookie = env.request.cookies["PREFS"]?
+ preferences = Preferences.from_json(URI.decode_www_form(prefs_cookie.value))
+ else
+ if language_header = env.request.headers["Accept-Language"]?
+ if language = ANG.language_negotiator.best(language_header, LOCALES.keys)
+ preferences.locale = language.header
+ end
+ end
+ end
+ rescue
+ preferences = Preferences.from_json("{}")
+ end
+
+ env.set "preferences", preferences
+ env.response.headers["X-XSS-Protection"] = "1; mode=block"
+ env.response.headers["X-Content-Type-Options"] = "nosniff"
+
+ # Only allow the pages at /embed/* to be embedded
+ if env.request.resource.starts_with?("/embed")
+ frame_ancestors = "'self' file: http: https:"
+ else
+ frame_ancestors = "'none'"
+ end
+
+ # TODO: Remove style-src's 'unsafe-inline', requires to remove all
+ # inline styles (, style=" [..] ")
+ env.response.headers["Content-Security-Policy"] = {
+ "default-src 'none'",
+ "script-src 'self'",
+ "style-src 'self' 'unsafe-inline'",
+ "img-src 'self' data:",
+ "font-src 'self' data:",
+ "connect-src 'self'",
+ "manifest-src 'self'",
+ "media-src 'self' blob:",
+ "child-src 'self' blob:",
+ "frame-src 'self'",
+ "frame-ancestors " + frame_ancestors,
+ }.join("; ")
+
+ env.response.headers["Referrer-Policy"] = "same-origin"
+
+ # Ask the chrom*-based browsers to disable FLoC
+ # See: https://blog.runcloud.io/google-floc/
+ env.response.headers["Permissions-Policy"] = "interest-cohort=()"
+
+ if (Kemal.config.ssl || CONFIG.https_only) && CONFIG.hsts
+ env.response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload"
+ end
+
+ return if {
+ "/sb/",
+ "/vi/",
+ "/s_p/",
+ "/yts/",
+ "/ggpht/",
+ "/api/manifest/",
+ "/videoplayback",
+ "/latest_version",
+ "/download",
+ }.any? { |r| env.request.resource.starts_with? r }
+
+ if env.request.cookies.has_key? "SID"
+ sid = env.request.cookies["SID"].value
+
+ if sid.starts_with? "v1:"
+ raise "Cannot use token as SID"
+ end
+
+ if email = Database::SessionIDs.select_email(sid)
+ user = Database::Users.select!(email: email)
+ csrf_token = generate_response(sid, {
+ ":authorize_token",
+ ":playlist_ajax",
+ ":signout",
+ ":subscription_ajax",
+ ":token_ajax",
+ ":watch_ajax",
+ }, HMAC_KEY, 1.week)
+
+ preferences = user.preferences
+ env.set "preferences", preferences
+
+ env.set "sid", sid
+ env.set "csrf_token", csrf_token
+ env.set "user", user
+ end
+ end
+
+ dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s
+ thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode.to_s
+ thin_mode = thin_mode == "true"
+ locale = env.params.query["hl"]? || preferences.locale
+
+ preferences.dark_mode = dark_mode
+ preferences.thin_mode = thin_mode
+ preferences.locale = locale
+ env.set "preferences", preferences
+
+ # Allow media resources to be loaded from google servers
+ # TODO: check if *.youtube.com can be removed
+ #
+ # `!preferences.local` has to be checked after setting and
+ # reading `preferences` from the "PREFS" cookie and
+ # saved user preferences from the database, otherwise
+ # `https://*.googlevideo.com:443 https://*.youtube.com:443`
+ # will not be set in the CSP header if
+ # `default_user_preferences.local` is set to true on the
+ # configuration file, causing preference “Proxy Videos”
+ # not to work while having it disabled and using medium quality.
+ if CONFIG.disabled?("local") || !preferences.local
+ env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"].gsub("media-src", "media-src https://*.googlevideo.com:443 https://*.youtube.com:443")
+ end
+
+ current_page = env.request.path
+ if env.request.query
+ query = HTTP::Params.parse(env.request.query.not_nil!)
+
+ if query["referer"]?
+ query["referer"] = get_referer(env, "/")
+ end
+
+ current_page += "?#{query}"
+ end
+
+ env.set "current_page", URI.encode_www_form(current_page)
+ end
+end
diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr
index cd2e3323..508aa3e4 100644
--- a/src/invidious/routes/channels.cr
+++ b/src/invidious/routes/channels.cr
@@ -1,27 +1,32 @@
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::Channels
+ # Redirection for unsupported routes ("tabs")
+ def self.redirect_home(env)
+ ucid = env.params.url["ucid"]
+ return env.redirect "/channel/#{URI.encode_www_form(ucid)}"
+ end
+
def self.home(env)
self.videos(env)
end
def self.videos(env)
data = self.fetch_basic_information(env)
- if !data.is_a?(Tuple)
- return data
- end
- locale, user, subscriptions, continuation, ucid, channel = data
+ return data if !data.is_a?(Tuple)
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
+ locale, user, subscriptions, continuation, ucid, channel = data
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
- sort_options = {"last", "oldest", "newest"}
sort_by ||= "last"
+ sort_options = {"last", "oldest", "newest"}
+
+ items, next_continuation = fetch_channel_playlists(
+ channel.ucid, channel.author, continuation, sort_by
+ )
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items.uniq! do |item|
if item.responds_to?(:title)
item.title
@@ -29,62 +34,224 @@ module Invidious::Routes::Channels
item.author
end
end
- items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
+ items = items.select(SearchPlaylist)
items.each(&.author = "")
else
- sort_options = {"newest", "oldest", "popular"}
- sort_by ||= "newest"
+ # Fetch items and continuation token
+ if channel.is_age_gated
+ sort_by = ""
+ sort_options = [] of String
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
+ items = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ items = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ sort_by ||= "newest"
+ sort_options = {"newest", "oldest", "popular"}
- count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+ items, next_continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ end
end
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
+ templated "channel"
+ end
+
+ def self.shorts(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if !channel.tabs.includes? "shorts"
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ if channel.is_age_gated
+ sort_by = ""
+ sort_options = [] of String
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
+ items = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ items = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ sort_options = {"newest", "oldest", "popular"}
+
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ end
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
+ templated "channel"
+ end
+
+ def self.streams(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if !channel.tabs.includes? "streams"
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ if channel.is_age_gated
+ sort_by = ""
+ sort_options = [] of String
+ begin
+ playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
+ items = get_playlist_videos(playlist, offset: 0)
+ rescue ex : InfoException
+ # playlist doesnt exist.
+ items = [] of PlaylistVideo
+ end
+ next_continuation = nil
+ else
+ sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+ sort_options = {"newest", "oldest", "popular"}
+
+ # Fetch items and continuation token
+ items, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ end
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
templated "channel"
end
def self.playlists(env)
data = self.fetch_basic_information(env)
- if !data.is_a?(Tuple)
- return data
- end
+ return data if !data.is_a?(Tuple)
+
locale, user, subscriptions, continuation, ucid, channel = data
sort_options = {"last", "oldest", "newest"}
sort_by = env.params.query["sort_by"]?.try &.downcase
- sort_by ||= "last"
if channel.auto_generated
return env.redirect "/channel/#{channel.ucid}"
end
- items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
- items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
+ items, next_continuation = fetch_channel_playlists(
+ channel.ucid, channel.author, continuation, (sort_by || "last")
+ )
+
+ items = items.select(SearchPlaylist)
items.each(&.author = "")
- templated "playlists"
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
+ templated "channel"
+ end
+
+ def self.podcasts(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ sort_by = ""
+ sort_options = [] of String
+
+ items, next_continuation = fetch_channel_podcasts(
+ channel.ucid, channel.author, continuation
+ )
+
+ items = items.select(SearchPlaylist)
+ items.each(&.author = "")
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Podcasts
+ templated "channel"
+ end
+
+ def self.releases(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ sort_by = ""
+ sort_options = [] of String
+
+ items, next_continuation = fetch_channel_releases(
+ channel.ucid, channel.author, continuation
+ )
+
+ items = items.select(SearchPlaylist)
+ items.each(&.author = "")
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Releases
+ templated "channel"
+ end
+
+ def self.courses(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ sort_by = ""
+ sort_options = [] of String
+
+ items, next_continuation = fetch_channel_courses(
+ channel.ucid, channel.author, continuation
+ )
+
+ items = items.select(SearchPlaylist)
+ items.each(&.author = "")
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Courses
+ templated "channel"
end
def self.community(env)
+ return env.redirect env.request.path.sub("posts", "community") if env.request.path.split("/").last == "posts"
+
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
return data
end
locale, user, subscriptions, continuation, ucid, channel = data
+ # redirect to post page
+ if lb = env.params.query["lb"]?
+ env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}"
+ end
+
thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode
thin_mode = thin_mode == "true"
continuation = env.params.query["continuation"]?
- # sort_by = env.params.query["sort_by"]?.try &.downcase
- if !channel.tabs.includes? "community"
+ if !channel.tabs.includes? "community" && "posts"
return env.redirect "/channel/#{channel.ucid}"
end
+ # TODO: support sort options for community posts
+ sort_by = ""
+ sort_options = [] of String
+
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
env.response.status_code = 500
error_message = ex.message
+ rescue ex : NotFoundException
+ env.response.status_code = 404
+ error_message = ex.message
rescue ex
return error_template(500, ex)
end
@@ -92,6 +259,64 @@ module Invidious::Routes::Channels
templated "community"
end
+ def self.post(env)
+ # /post/{postId}
+ id = env.params.url["id"]
+ ucid = env.params.query["ucid"]?
+
+ prefs = env.get("preferences").as(Preferences)
+
+ locale = prefs.locale
+
+ thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode
+ thin_mode = thin_mode == "true"
+
+ nojs = env.params.query["nojs"]?
+
+ nojs ||= "0"
+ nojs = nojs == "1"
+
+ if !ucid.nil?
+ ucid = ucid.to_s
+ post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
+ else
+ # resolve the url to get the author's UCID
+ response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
+ return error_template(400, "Invalid post ID") if response["error"]?
+
+ ucid = response.dig("endpoint", "browseEndpoint", "browseId").as_s
+ post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
+ end
+
+ post_response = JSON.parse(post_response)
+
+ if nojs
+ comments = Comments.fetch_community_post_comments(ucid, id)
+ comment_html = JSON.parse(Comments.parse_youtube(id, comments, "html", locale, thin_mode, is_post: true))["contentHtml"]
+ end
+ templated "post"
+ end
+
+ def self.channels(env)
+ data = self.fetch_basic_information(env)
+ return data if !data.is_a?(Tuple)
+
+ locale, user, subscriptions, continuation, ucid, channel = data
+
+ if channel.auto_generated
+ return env.redirect "/channel/#{channel.ucid}"
+ end
+
+ items, next_continuation = fetch_related_channels(channel, continuation)
+
+ # Featured/related channels can't be sorted
+ sort_options = [] of String
+ sort_by = nil
+
+ selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
+ templated "channel"
+ end
+
def self.about(env)
data = self.fetch_basic_information(env)
if !data.is_a?(Tuple)
@@ -102,6 +327,12 @@ module Invidious::Routes::Channels
env.redirect "/channel/#{ucid}"
end
+ private KNOWN_TABS = {
+ "home", "videos", "shorts", "streams", "podcasts",
+ "releases", "courses", "playlists", "community", "channels", "about",
+ "posts",
+ }
+
# Redirects brand url channels to a normal /channel/:ucid route
def self.brand_redirect(env)
locale = env.get("preferences").as(Preferences).locale
@@ -112,23 +343,29 @@ module Invidious::Routes::Channels
yt_url_params = URI::Params.encode(env.params.query.to_h.select(["a", "u", "user"]))
# Retrieves URL params that only Invidious uses
- invidious_url_params = URI::Params.encode(env.params.query.to_h.select!(["a", "u", "user"]))
+ invidious_url_params = env.params.query.dup
+ invidious_url_params.delete_all("a")
+ invidious_url_params.delete_all("u")
+ invidious_url_params.delete_all("user")
begin
resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}")
ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"]
rescue ex : InfoException | KeyError
- raise InfoException.new(translate(locale, "This channel does not exist."))
+ return error_template(404, translate(locale, "This channel does not exist."))
end
- selected_tab = env.request.path.split("/")[-1]
- if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
+ selected_tab = env.params.url["tab"]?
+
+ if KNOWN_TABS.includes? selected_tab
url = "/channel/#{ucid}/#{selected_tab}"
else
url = "/channel/#{ucid}"
end
- env.redirect url
+ url += "?#{invidious_url_params}" if !invidious_url_params.empty?
+
+ return env.redirect url
end
# Handles redirects for the /profile endpoint
@@ -141,7 +378,7 @@ module Invidious::Routes::Channels
user = env.params.query["user"]?
if !user
- raise InfoException.new("This channel does not exist.")
+ return error_template(404, "This channel does not exist.")
else
env.redirect "/user/#{user}#{uri_params}"
end
@@ -197,10 +434,13 @@ module Invidious::Routes::Channels
channel = get_about_info(ucid, locale)
rescue ex : ChannelRedirect
return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
+ env.set "search", "channel:#{ucid} "
return {locale, user, subscriptions, continuation, ucid, channel}
end
end
diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr
index 207970b0..930e4915 100644
--- a/src/invidious/routes/embed.cr
+++ b/src/invidious/routes/embed.cr
@@ -2,16 +2,25 @@
module Invidious::Routes::Embed
def self.redirect(env)
+ locale = env.get("preferences").as(Preferences).locale
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
+ if videos.empty?
+ url = "/playlist?list=#{plid}"
+ raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
+ end
+
+ first_playlist_video = videos[0].as(PlaylistVideo)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{videos[0].id}?#{env.params.query}"
+ url = "/embed/#{first_playlist_video}?#{env.params.query}"
if env.params.query.size > 0
url += "?#{env.params.query}"
@@ -24,6 +33,7 @@ module Invidious::Routes::Embed
end
def self.show(env)
+ locale = env.get("preferences").as(Preferences).locale
id = env.params.url["id"]
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
@@ -60,11 +70,19 @@ module Invidious::Routes::Embed
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
+ if videos.empty?
+ url = "/playlist?list=#{plid}"
+ raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
+ end
+
+ first_playlist_video = videos[0].as(PlaylistVideo)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{videos[0].id}"
+ url = "/embed/#{first_playlist_video.id}"
elsif video_series
url = "/embed/#{video_series.shift}"
env.params.query["playlist"] = video_series.join(",")
@@ -117,8 +135,8 @@ module Invidious::Routes::Embed
begin
video = get_video(id, region: params.region)
- rescue ex : VideoRedirect
- return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
@@ -133,7 +151,7 @@ module Invidious::Routes::Embed
# PG_DB.exec("UPDATE users SET watched = array_append(watched, $1) WHERE email = $2", id, user.as(User).email)
# end
- if notifications && notifications.includes? id
+ if CONFIG.enable_user_notifications && notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
@@ -143,10 +161,12 @@ module Invidious::Routes::Embed
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
end
+ # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -187,6 +207,14 @@ module Invidious::Routes::Embed
return env.redirect url
end
+ if CONFIG.invidious_companion.present?
+ invidious_companion = CONFIG.invidious_companion.sample
+ env.response.headers["Content-Security-Policy"] =
+ env.response.headers["Content-Security-Policy"]
+ .gsub("media-src", "media-src #{invidious_companion.public_url}")
+ .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
+ end
+
rendered "embed"
end
end
diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr
new file mode 100644
index 00000000..1e9ab44e
--- /dev/null
+++ b/src/invidious/routes/errors.cr
@@ -0,0 +1,52 @@
+module Invidious::Routes::ErrorRoutes
+ def self.error_404(env)
+ # Workaround for #3117
+ if HOST_URL.empty? && env.request.path.starts_with?("/v1/storyboards/sb")
+ return env.redirect "#{env.request.path[15..]}?#{env.params.query}"
+ end
+
+ if md = env.request.path.match(/^\/(?([a-zA-Z0-9_-]{11})|(\w+))$/)
+ item = md["id"]
+
+ # Check if item is branding URL e.g. https://youtube.com/gaming
+ response = YT_POOL.client &.get("/#{item}")
+
+ if response.status_code == 301
+ response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target)
+ end
+
+ if response.body.empty?
+ env.response.headers["Location"] = "/"
+ haltf env, status_code: 302
+ end
+
+ html = XML.parse_html(response.body)
+ ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
+
+ if ucid
+ env.response.headers["Location"] = "/channel/#{ucid}"
+ haltf env, status_code: 302
+ end
+
+ params = [] of String
+ env.params.query.each do |k, v|
+ params << "#{k}=#{v}"
+ end
+ params = params.join("&")
+
+ url = "/watch?v=#{item}"
+ if !params.empty?
+ url += "&#{params}"
+ end
+
+ # Check if item is video ID
+ if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
+ env.response.headers["Location"] = url
+ haltf env, status_code: 302
+ end
+ end
+
+ env.response.headers["Location"] = "/"
+ haltf env, status_code: 302
+ end
+end
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index f7f7b426..070c96eb 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -83,10 +83,6 @@ module Invidious::Routes::Feeds
headers = HTTP::Headers.new
headers["Cookie"] = env.request.headers["Cookie"]
- if !user.password
- user, sid = get_user(sid, headers)
- end
-
max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE)
max_results ||= user.preferences.max_results
max_results ||= CONFIG.default_user_preferences.max_results
@@ -96,14 +92,20 @@ module Invidious::Routes::Feeds
videos, notifications = get_subscription_feed(user, max_results, page)
- # "updated" here is used for delivering new notifications, so if
- # we know a user has looked at their feed e.g. in the past 10 minutes,
- # they've already seen a video posted 20 minutes ago, and don't need
- # to be notified.
- Invidious::Database::Users.clear_notifications(user)
- user.notifications = [] of String
+ if CONFIG.enable_user_notifications
+ # "updated" here is used for delivering new notifications, so if
+ # we know a user has looked at their feed e.g. in the past 10 minutes,
+ # they've already seen a video posted 20 minutes ago, and don't need
+ # to be notified.
+ Invidious::Database::Users.clear_notifications(user)
+ user.notifications = [] of String
+ end
env.set "user", user
+ # Used for pagination links
+ base_url = "/feed/subscriptions"
+ base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
+
templated "feeds/subscriptions"
end
@@ -131,78 +133,91 @@ module Invidious::Routes::Feeds
end
watched ||= [] of String
+ # Used for pagination links
+ base_url = "/feed/history"
+ base_url += "?max_results=#{max_results}" if env.params.query.has_key?("max_results")
+
templated "feeds/history"
end
# RSS feeds
def self.rss_channel(env)
- locale = env.get("preferences").as(Preferences).locale
-
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
- ucid = env.params.url["ucid"]
+ if env.params.url["ucid"].matches?(/^[\w-]+$/)
+ ucid = env.params.url["ucid"]
+ else
+ return error_atom(400, InfoException.new("Invalid channel ucid provided."))
+ end
params = HTTP::Params.parse(env.params.query["params"]? || "")
- begin
- channel = get_about_info(ucid, locale)
- rescue ex : ChannelRedirect
- return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
- rescue ex
- return error_atom(500, ex)
- end
+ namespaces = {
+ "yt" => "http://www.youtube.com/xml/schemas/2015",
+ "media" => "http://search.yahoo.com/mrss/",
+ "default" => "http://www.w3.org/2005/Atom",
+ }
- response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
- rss = XML.parse_html(response.body)
+ response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
+ return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
+ rss = XML.parse(response.body)
- videos = rss.xpath_nodes("//feed/entry").map do |entry|
- video_id = entry.xpath_node("videoid").not_nil!.content
- title = entry.xpath_node("title").not_nil!.content
+ videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
+ video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+ title = entry.xpath_node("default:title", namespaces).not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
+ published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
- author = entry.xpath_node("author/name").not_nil!.content
- ucid = entry.xpath_node("channelid").not_nil!.content
- description_html = entry.xpath_node("group/description").not_nil!.to_s
- views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
+ author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
+ video_ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+ description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s
+ views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64
SearchVideo.new({
title: title,
id: video_id,
author: author,
- ucid: ucid,
+ ucid: video_ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
- live_now: false,
- paid: false,
- premium: false,
premiere_timestamp: nil,
+ author_verified: false,
+ author_thumbnail: nil,
+ badges: VideoBadges::None,
})
end
+ author = ""
+ author = videos[0].author if videos.size > 0
+
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
- xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
- xml.element("yt:channelId") { xml.text channel.ucid }
- xml.element("icon") { xml.text channel.author_thumbnail }
- xml.element("title") { xml.text channel.author }
- xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
+ xml.element("id") { xml.text "yt:channel:#{ucid}" }
+ xml.element("yt:channelId") { xml.text ucid }
+ xml.element("title") { xml.text author }
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{ucid}")
xml.element("author") do
- xml.element("name") { xml.text channel.author }
- xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
+ xml.element("name") { xml.text author }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
+ end
+
+ xml.element("image") do
+ xml.element("url") { xml.text "" }
+ xml.element("title") { xml.text author }
+ xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
end
videos.each do |video|
- video.to_xml(channel.auto_generated, params, xml)
+ video.to_xml(false, params, xml)
end
end
end
@@ -281,7 +296,13 @@ module Invidious::Routes::Feeds
xml.element("name") { xml.text playlist.author }
end
- videos.each &.to_xml(xml)
+ videos.each do |video|
+ if video.is_a? PlaylistVideo
+ video.to_xml(xml)
+ else
+ video.to_xml(env, locale, xml)
+ end
+ end
end
end
else
@@ -290,8 +311,9 @@ module Invidious::Routes::Feeds
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
- document = XML.parse(response.body)
+ return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
+ document = XML.parse(response.body)
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
node.attributes.each do |attribute|
case attribute.name
@@ -386,22 +408,23 @@ module Invidious::Routes::Feeds
end
spawn do
- rss = XML.parse_html(body)
- rss.xpath_nodes("//feed/entry").each do |entry|
- id = entry.xpath_node("videoid").not_nil!.content
- author = entry.xpath_node("author/name").not_nil!.content
- published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
- updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
+ # TODO: unify this with the other almost identical looking parts in this and channels.cr somehow?
+ namespaces = {
+ "yt" => "http://www.youtube.com/xml/schemas/2015",
+ "default" => "http://www.w3.org/2005/Atom",
+ }
+ rss = XML.parse(body)
+ rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
+ id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+ author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
+ published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
+ updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
- video = get_video(id, force_refresh: true)
-
- # Deliver notifications to `/api/v1/auth/notifications`
- payload = {
- "topic" => video.ucid,
- "videoId" => video.id,
- "published" => published.to_unix,
- }.to_json
- PG_DB.exec("NOTIFY notifications, E'#{payload}'")
+ begin
+ video = get_video(id, force_refresh: true)
+ rescue
+ next # skip this video since it raised an exception (e.g. it is a scheduled live event)
+ end
video = ChannelVideo.new({
id: id,
@@ -417,7 +440,9 @@ module Invidious::Routes::Feeds
})
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
- Invidious::Database::Users.add_notification(video) if was_insert
+ if was_insert
+ NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
+ end
end
end
diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 594a7869..51d85dfe 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -3,17 +3,7 @@ module Invidious::Routes::Images
def self.ggpht(env)
url = env.request.path.lchop("/ggpht")
- headers = (
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- HTTP::Headers{":authority" => "yt3.ggpht.com"}
- else
- HTTP::Headers.new
- end
- {% else %}
- HTTP::Headers.new
- {% end %}
- )
+ headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
@@ -21,43 +11,10 @@ module Invidious::Routes::Images
end
end
- # We're encapsulating this into a proc in order to easily reuse this
- # portion of the code for each request block below.
- request_proc = ->(response : HTTP::Client::Response) {
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300
- env.response.headers.delete("Transfer-Encoding")
- return
- end
-
- proxy_file(response, env)
- }
-
begin
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- YT_POOL.client &.get(url, headers) do |resp|
- return request_proc.call(resp)
- end
- else
- HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- {% end %}
+ GGPHT_POOL.client &.get(url, headers) do |resp|
+ return self.proxy_image(env, resp)
+ end
rescue ex
end
end
@@ -78,51 +35,17 @@ module Invidious::Routes::Images
headers = HTTP::Headers.new
- {% unless flag?(:disable_quic) %}
- headers[":authority"] = "#{authority}.ytimg.com"
- {% end %}
-
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
headers[header] = env.request.headers[header]
end
end
- request_proc = ->(response : HTTP::Client::Response) {
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Connection"] = "close"
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- proxy_file(response, env)
- }
-
begin
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- YT_POOL.client &.get(url, headers) do |resp|
- return request_proc.call(resp)
- end
- else
- HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- {% end %}
+ get_ytimg_pool(authority).client &.get(url, headers) do |resp|
+ env.response.headers["Connection"] = "close"
+ return self.proxy_image(env, resp)
+ end
rescue ex
end
end
@@ -133,17 +56,7 @@ module Invidious::Routes::Images
name = env.params.url["name"]
url = env.request.resource
- headers = (
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- HTTP::Headers{":authority" => "i9.ytimg.com"}
- else
- HTTP::Headers.new
- end
- {% else %}
- HTTP::Headers.new
- {% end %}
- )
+ headers = HTTP::Headers.new
REQUEST_HEADERS_WHITELIST.each do |header|
if env.request.headers[header]?
@@ -151,40 +64,10 @@ module Invidious::Routes::Images
end
end
- request_proc = ->(response : HTTP::Client::Response) {
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300 && response.status_code != 404
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- proxy_file(response, env)
- }
-
begin
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- YT_POOL.client &.get(url, headers) do |resp|
- return request_proc.call(resp)
- end
- else
- HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- {% end %}
+ get_ytimg_pool("i9").client &.get(url, headers) do |resp|
+ return self.proxy_image(env, resp)
+ end
rescue ex
end
end
@@ -223,41 +106,15 @@ module Invidious::Routes::Images
id = env.params.url["id"]
name = env.params.url["name"]
- headers = (
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- HTTP::Headers{":authority" => "i.ytimg.com"}
- else
- HTTP::Headers.new
- end
- {% else %}
- HTTP::Headers.new
- {% end %}
- )
+ headers = HTTP::Headers.new
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
- # Logic here is short enough that manually typing them out should be fine.
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200
- name = thumb[:url] + ".jpg"
- break
- end
- else
- if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
- name = thumb[:url] + ".jpg"
- break
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
- name = thumb[:url] + ".jpg"
- break
- end
- {% end %}
+ if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
+ name = thumb[:url] + ".jpg"
+ break
+ end
end
end
@@ -269,41 +126,28 @@ module Invidious::Routes::Images
end
end
- request_proc = ->(response : HTTP::Client::Response) {
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300 && response.status_code != 404
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- proxy_file(response, env)
- }
-
begin
- {% unless flag?(:disable_quic) %}
- if CONFIG.use_quic
- YT_POOL.client &.get(url, headers) do |resp|
- return request_proc.call(resp)
- end
- else
- HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- end
- {% else %}
- # This can likely be optimized into a (small) pool sometime in the future.
- HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
- return request_proc.call(resp)
- end
- {% end %}
+ get_ytimg_pool("i").client &.get(url, headers) do |resp|
+ return self.proxy_image(env, resp)
+ end
rescue ex
end
end
+
+ private def self.proxy_image(env, response)
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ return proxy_file(response, env)
+ end
end
diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr
index 99fc13a2..e7de5018 100644
--- a/src/invidious/routes/login.cr
+++ b/src/invidious/routes/login.cr
@@ -6,14 +6,14 @@ module Invidious::Routes::Login
user = env.get? "user"
- return env.redirect "/feed/subscriptions" if user
+ referer = get_referer(env, "/feed/subscriptions")
+
+ return env.redirect referer if user
if !CONFIG.login_enabled
return error_template(400, "Login has been disabled by administrator.")
end
- referer = get_referer(env, "/feed/subscriptions")
-
email = nil
password = nil
captcha = nil
@@ -21,12 +21,6 @@ module Invidious::Routes::Login
account_type = env.params.query["type"]?
account_type ||= "invidious"
- captcha_type = env.params.query["captcha"]?
- captcha_type ||= "image"
-
- tfa = env.params.query["tfa"]?
- prompt = nil
-
templated "user/login"
end
@@ -47,283 +41,18 @@ module Invidious::Routes::Login
account_type ||= "invidious"
case account_type
- when "google"
- tfa_code = env.params.body["tfa"]?.try &.lchop("G-")
- traceback = IO::Memory.new
-
- # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
- begin
- client = nil # Declare variable
- {% unless flag?(:disable_quic) %}
- client = CONFIG.use_quic ? QUIC::Client.new(LOGIN_URL) : HTTP::Client.new(LOGIN_URL)
- {% else %}
- client = HTTP::Client.new(LOGIN_URL)
- {% end %}
-
- headers = HTTP::Headers.new
-
- login_page = client.get("/ServiceLogin")
- headers = login_page.cookies.add_request_headers(headers)
-
- lookup_req = {
- email, nil, [] of String, nil, "US", nil, nil, 2, false, true,
- {nil, nil,
- {2, 1, nil, 1,
- "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
- nil, [] of String, 4},
- 1,
- {nil, nil, [] of String},
- nil, nil, nil, true,
- },
- email,
- }.to_json
-
- traceback << "Getting lookup..."
-
- headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
- headers["Google-Accounts-XSRF"] = "1"
-
- response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
- lookup_results = JSON.parse(response.body[5..-1])
-
- traceback << "done, returned #{response.status_code}.
"
-
- user_hash = lookup_results[0][2]
-
- if token = env.params.body["token"]?
- answer = env.params.body["answer"]?
- captcha = {token, answer}
- else
- captcha = nil
- end
-
- challenge_req = {
- user_hash, nil, 1, nil,
- {1, nil, nil, nil,
- {password, captcha, true},
- },
- {nil, nil,
- {2, 1, nil, 1,
- "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn",
- nil, [] of String, 4},
- 1,
- {nil, nil, [] of String},
- nil, nil, nil, true,
- },
- }.to_json
-
- traceback << "Getting challenge..."
-
- response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req))
- headers = response.cookies.add_request_headers(headers)
- challenge_results = JSON.parse(response.body[5..-1])
-
- traceback << "done, returned #{response.status_code}.
"
-
- headers["Cookie"] = URI.decode_www_form(headers["Cookie"])
-
- if challenge_results[0][3]?.try &.== 7
- return error_template(423, "Account has temporarily been disabled")
- end
-
- if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s
- account_type = "google"
- captcha_type = "image"
- prompt = nil
- tfa = tfa_code
- captcha = {tokens: [token], question: ""}
-
- return templated "user/login"
- end
-
- if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED"
- return error_template(401, "Incorrect password")
- end
-
- prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]?
- if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type
- traceback << "Handling prompt #{prompt_type}.
"
- case prompt_type
- when "TWO_STEP_VERIFICATION"
- prompt_type = 2
- else # "LOGIN_CHALLENGE"
- prompt_type = 4
- end
-
- # Prefer Authenticator app and SMS over unsupported protocols
- if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2
- tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0]
-
- traceback << "Selecting challenge #{tfa[8]}..."
- select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json
-
- tl = challenge_results[1][2]
-
- tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body
- tfa = tfa[5..-1]
- tfa = JSON.parse(tfa)[0][-1]
-
- traceback << "done.
"
- else
- traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.
"
- tfa = challenge_results[0][-1][0][0]
- end
-
- if tfa[5] == "QUOTA_EXCEEDED"
- return error_template(423, "Quota exceeded, try again in a few hours")
- end
-
- if !tfa_code
- account_type = "google"
- captcha_type = "image"
-
- case tfa[8]
- when 6, 9
- prompt = "Google verification code"
- when 12
- prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
- when 15
- prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}"
- else
- prompt = "Google verification code"
- end
-
- tfa = nil
- captcha = nil
- return templated "user/login"
- end
-
- tl = challenge_results[1][2]
-
- request_type = tfa[8]
- case request_type
- when 6 # Authenticator app
- tfa_req = {
- user_hash, nil, 2, nil,
- {6, nil, nil, nil, nil,
- {tfa_code, false},
- },
- }.to_json
- when 9 # Voice or text message
- tfa_req = {
- user_hash, nil, 2, nil,
- {9, nil, nil, nil, nil, nil, nil, nil,
- {nil, tfa_code, false, 2},
- },
- }.to_json
- when 12 # Recovery email
- tfa_req = {
- user_hash, nil, 4, nil,
- {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
- {tfa_code},
- },
- }.to_json
- when 15 # Security question
- tfa_req = {
- user_hash, nil, 5, nil,
- {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
- {tfa_code},
- },
- }.to_json
- else
- return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.")
- end
-
- traceback << "Submitting challenge..."
-
- response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req))
- headers = response.cookies.add_request_headers(headers)
- challenge_results = JSON.parse(response.body[5..-1])
-
- if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") ||
- (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT")
- return error_template(401, "Invalid TFA code")
- end
-
- traceback << "done.
"
- end
-
- traceback << "Logging in..."
-
- location = URI.parse(challenge_results[0][-1][2].to_s)
- cookies = HTTP::Cookies.from_client_headers(headers)
-
- headers.delete("Content-Type")
- headers.delete("Google-Accounts-XSRF")
-
- loop do
- if !location || location.path == "/ManageAccount"
- break
- end
-
- # Occasionally there will be a second page after login confirming
- # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle.
-
- if location.path.starts_with? "/b/0/SmsAuthInterstitial"
- traceback << "Unhandled dialog /b/0/SmsAuthInterstitial."
- end
-
- login = client.get(location.request_target, headers)
-
- headers = login.cookies.add_request_headers(headers)
- location = login.headers["Location"]?.try { |u| URI.parse(u) }
- end
-
- cookies = HTTP::Cookies.from_client_headers(headers)
- sid = cookies["SID"]?.try &.value
- if !sid
- raise "Couldn't get SID."
- end
-
- user, sid = get_user(sid, headers)
-
- # We are now logged in
- traceback << "done.
"
-
- host = URI.parse(env.request.headers["Host"]).host
-
- cookies.each do |cookie|
- cookie.secure = Invidious::User::Cookies::SECURE
-
- if cookie.extension
- cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host)
- cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "")
- end
- env.response.cookies << cookie
- end
-
- if env.request.cookies["PREFS"]?
- user.preferences = env.get("preferences").as(Preferences)
- Invidious::Database::Users.update_preferences(user)
-
- cookie = env.request.cookies["PREFS"]
- cookie.expires = Time.utc(1990, 1, 1)
- env.response.cookies << cookie
- end
-
- env.redirect referer
- rescue ex
- traceback.rewind
- # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.")
- error_message = %(#{ex.message}
Traceback:
#{traceback.gets_to_end})
- return error_template(500, error_message)
- end
when "invidious"
- if !email
+ if email.nil? || email.empty?
return error_template(401, "User ID is a required field")
end
- if !password
+ if password.nil? || password.empty?
return error_template(401, "Password is a required field")
end
user = Invidious::Database::Users.select(email: email)
if user
- if !user.password
- return error_template(400, "Please sign in using 'Log in with Google'")
- end
-
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
Invidious::Database::SessionIDs.insert(sid, email)
@@ -356,36 +85,14 @@ module Invidious::Routes::Login
password = password.byte_slice(0, 55)
if CONFIG.captcha_enabled
- captcha_type = env.params.body["captcha_type"]?
answer = env.params.body["answer"]?
- change_type = env.params.body["change_type"]?
- if !captcha_type || change_type
- if change_type
- captcha_type = change_type
- end
- captcha_type ||= "image"
-
- account_type = "invidious"
- tfa = false
- prompt = ""
-
- if captcha_type == "image"
- captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
- else
- captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
- end
-
- return templated "user/login"
- end
+ account_type = "invidious"
+ captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
- answer ||= ""
- captcha_type ||= "image"
-
- case captcha_type
- when "image"
+ if answer
answer = answer.lstrip('0')
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
@@ -394,27 +101,8 @@ module Invidious::Routes::Login
rescue ex
return error_template(400, ex)
end
- else # "text"
- answer = Digest::MD5.hexdigest(answer.downcase.strip)
-
- if tokens.empty?
- return error_template(500, "Erroneous CAPTCHA")
- end
-
- found_valid_captcha = false
- error_exception = Exception.new
- tokens.each do |tok|
- begin
- validate_request(tok, answer, env.request, HMAC_KEY, locale)
- found_valid_captcha = true
- rescue ex
- error_exception = ex
- end
- end
-
- if !found_valid_captcha
- return error_template(500, error_exception)
- end
+ else
+ return templated "user/login"
end
end
@@ -481,11 +169,4 @@ module Invidious::Routes::Login
env.redirect referer
end
-
- def self.captcha(env)
- headers = HTTP::Headers{":authority" => "accounts.google.com"}
- response = YT_POOL.client &.get(env.request.resource, headers)
- env.response.headers["Content-Type"] = response.headers["Content-Type"]
- response.body
- end
end
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index d6bd9571..0b868755 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -40,7 +40,21 @@ module Invidious::Routes::Misc
def self.cross_instance_redirect(env)
referer = get_referer(env)
- instance_url = fetch_random_instance
+
+ instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
+ # Filter out the current instance
+ other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
+
+ if other_available_instances.empty?
+ # If the current instance is the only one, use the redirect URL as fallback
+ instance_url = "redirect.invidious.io"
+ else
+ # Select other random instance
+ # Sample returns an array
+ # Instances are packaged as {region, domain} in the instance list
+ instance_url = other_available_instances.sample(1)[0][1]
+ end
+
env.redirect "https://#{instance_url}#{referer}"
end
end
diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr
index 272a3dc7..8922b740 100644
--- a/src/invidious/routes/notifications.cr
+++ b/src/invidious/routes/notifications.cr
@@ -24,50 +24,6 @@ module Invidious::Routes::Notifications
user = user.as(User)
- if !user.password
- channel_req = {} of String => String
-
- channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true"
- channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || ""
- channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true"
-
- channel_req.reject! { |k, v| v != "true" && v != "false" }
-
- headers = HTTP::Headers.new
- headers["Cookie"] = env.request.headers["Cookie"]
-
- html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
-
- cookies = HTTP::Cookies.from_client_headers(headers)
- html.cookies.each do |cookie|
- if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
- if cookies[cookie.name]?
- cookies[cookie.name] = cookie
- else
- cookies << cookie
- end
- end
- end
- headers = cookies.add_request_headers(headers)
-
- if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
- session_token = match["session_token"]
- else
- return env.redirect referer
- end
-
- headers["content-type"] = "application/x-www-form-urlencoded"
- channel_req["session_token"] = session_token
-
- subs = XML.parse_html(html.body)
- subs.xpath_nodes(%q(//a[@class="subscription-title yt-uix-sessionlink"]/@href)).each do |channel|
- channel_id = channel.content.lstrip("/channel/").not_nil!
- channel_req["channel_id"] = channel_id
-
- YT_POOL.client &.post("/subscription_ajax?action_update_subscription_preferences=1", headers, form: channel_req)
- end
- end
-
if redirect
env.redirect referer
else
diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr
index dbeb4f97..f2213da4 100644
--- a/src/invidious/routes/playlists.cr
+++ b/src/invidious/routes/playlists.cr
@@ -66,7 +66,13 @@ module Invidious::Routes::Playlists
user = user.as(User)
playlist_id = env.params.query["list"]
- playlist = get_playlist(playlist_id)
+ begin
+ playlist = get_playlist(playlist_id)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
+ rescue ex
+ return error_template(500, ex)
+ end
subscribe_playlist(user, playlist)
env.redirect "/playlist?list=#{playlist.id}"
@@ -157,13 +163,20 @@ module Invidious::Routes::Playlists
end
begin
- videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
+ items = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
- videos = [] of PlaylistVideo
+ items = [] of PlaylistVideo
end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY)
+ # Pagination
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/playlist?list=#{playlist.id}",
+ current_page: page,
+ show_next: (items.size == 100)
+ )
+
templated "edit_playlist"
end
@@ -212,7 +225,10 @@ module Invidious::Routes::Playlists
end
def self.add_playlist_items_page(env)
- locale = env.get("preferences").as(Preferences).locale
+ prefs = env.get("preferences").as(Preferences)
+ locale = prefs.locale
+
+ region = env.params.query["region"]? || prefs.region
user = env.get? "user"
sid = env.get? "sid"
@@ -236,18 +252,21 @@ module Invidious::Routes::Playlists
return env.redirect referer
end
- query = env.params.query["q"]?
- if query
- begin
- search_query, items, operators = process_search_query(query, page, user, region: nil)
- videos = items.select(SearchVideo).map(&.as(SearchVideo))
- rescue ex
- videos = [] of SearchVideo
- end
- else
- videos = [] of SearchVideo
+ begin
+ query = Invidious::Search::Query.new(env.params.query, :playlist, region)
+ items = query.process.select(SearchVideo).map(&.as(SearchVideo))
+ rescue ex
+ items = [] of SearchVideo
end
+ # Pagination
+ query_encoded = URI.encode_www_form(query.try &.text || "", space_to_plus: true)
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/add_playlist_items?list=#{playlist.id}&q=#{query_encoded}",
+ current_page: page,
+ show_next: (items.size >= 20)
+ )
+
env.set "add_playlist_items", plid
templated "add_playlist_items"
end
@@ -285,27 +304,12 @@ module Invidious::Routes::Playlists
end
end
- if env.params.query["action_create_playlist"]?
- action = "action_create_playlist"
- elsif env.params.query["action_delete_playlist"]?
- action = "action_delete_playlist"
- elsif env.params.query["action_edit_playlist"]?
- action = "action_edit_playlist"
- elsif env.params.query["action_add_video"]?
- action = "action_add_video"
- video_id = env.params.query["video_id"]
- elsif env.params.query["action_remove_video"]?
- action = "action_remove_video"
- elsif env.params.query["action_move_video_before"]?
- action = "action_move_video_before"
- else
- return env.redirect referer
- end
-
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
raise "Invalid user" if playlist.author != user.email
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
if redirect
return error_template(400, ex)
@@ -314,21 +318,13 @@ module Invidious::Routes::Playlists
end
end
- if !user.password
- # TODO: Playlist stub, sync with YouTube for Google accounts
- # playlist_ajax(playlist_id, action, env.request.headers)
- end
- email = user.email
-
- case action
- when "action_edit_playlist"
- # TODO: Playlist stub
- when "action_add_video"
- if playlist.index.size >= 500
+ case action = env.params.query["action"]?
+ when "add_video"
+ if playlist.index.size >= CONFIG.playlist_length_limit
if redirect
- return error_template(400, "Playlist cannot have more than 500 videos")
+ return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
else
- return error_json(400, "Playlist cannot have more than 500 videos")
+ return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
end
@@ -336,6 +332,8 @@ module Invidious::Routes::Playlists
begin
video = get_video(video_id)
+ rescue ex : NotFoundException
+ return error_json(404, ex)
rescue ex
if redirect
return error_template(500, ex)
@@ -358,12 +356,14 @@ module Invidious::Routes::Playlists
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
- when "action_remove_video"
+ when "remove_video"
index = env.params.query["set_video_id"]
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(playlist_id, index)
- when "action_move_video_before"
+ when "move_video_before"
# TODO: Playlist stub
+ when nil
+ return error_json(400, "Missing action")
else
return error_json(400, "Unsupported action #{action}")
end
@@ -396,12 +396,19 @@ module Invidious::Routes::Playlists
begin
playlist = get_playlist(plid)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- page_count = (playlist.video_count / 100).to_i
- page_count += 1 if (playlist.video_count % 100) > 0
+ if playlist.is_a? InvidiousPlaylist
+ page_count = (playlist.video_count / 100).to_i
+ page_count += 1 if (playlist.video_count % 100) > 0
+ else
+ page_count = (playlist.video_count / 200).to_i
+ page_count += 1 if (playlist.video_count % 200) > 0
+ end
if page > page_count
return env.redirect "/playlist?list=#{plid}&page=#{page_count}"
@@ -412,7 +419,11 @@ module Invidious::Routes::Playlists
end
begin
- videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
+ if playlist.is_a? InvidiousPlaylist
+ items = get_playlist_videos(playlist, offset: (page - 1) * 100)
+ else
+ items = get_playlist_videos(playlist, offset: (page - 1) * 200)
+ end
rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}")
end
@@ -421,6 +432,13 @@ module Invidious::Routes::Playlists
env.set "remove_playlist_items", plid
end
+ # Pagination
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/playlist?list=#{playlist.id}",
+ current_page: page,
+ show_next: (page_count != 1 && page < page_count)
+ )
+
templated "playlist"
end
diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr
index 570cba69..39ca77c0 100644
--- a/src/invidious/routes/preferences.cr
+++ b/src/invidious/routes/preferences.cr
@@ -27,6 +27,10 @@ module Invidious::Routes::PreferencesRoute
annotations_subscribed ||= "off"
annotations_subscribed = annotations_subscribed == "on"
+ preload = env.params.body["preload"]?.try &.as(String)
+ preload ||= "off"
+ preload = preload == "on"
+
autoplay = env.params.body["autoplay"]?.try &.as(String)
autoplay ||= "off"
autoplay = autoplay == "on"
@@ -144,6 +148,7 @@ module Invidious::Routes::PreferencesRoute
preferences = Preferences.from_json({
annotations: annotations,
annotations_subscribed: annotations_subscribed,
+ preload: preload,
autoplay: autoplay,
captions: captions,
comments: comments,
@@ -214,7 +219,7 @@ module Invidious::Routes::PreferencesRoute
statistics_enabled ||= "off"
CONFIG.statistics_enabled = statistics_enabled == "on"
- CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
+ CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
File.write("config/config.yml", CONFIG.to_yaml)
end
@@ -310,6 +315,24 @@ module Invidious::Routes::PreferencesRoute
response: error_template(415, "Invalid subscription file uploaded")
)
end
+ when "import_youtube_pl"
+ filename = part.filename || ""
+ success = Invidious::User::Import.from_youtube_pl(user, body, filename, type)
+
+ if !success
+ haltf(env, status_code: 415,
+ response: error_template(415, "Invalid playlist file uploaded")
+ )
+ end
+ when "import_youtube_wh"
+ filename = part.filename || ""
+ success = Invidious::User::Import.from_youtube_wh(user, body, filename, type)
+
+ if !success
+ haltf(env, status_code: 415,
+ response: error_template(415, "Invalid watch history file uploaded")
+ )
+ end
when "import_freetube"
Invidious::User::Import.from_freetube(user, body)
when "import_newpipe_subscriptions"
diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr
index 3f4c7e5e..b195c7b3 100644
--- a/src/invidious/routes/search.cr
+++ b/src/invidious/routes/search.cr
@@ -37,38 +37,87 @@ module Invidious::Routes::Search
end
def self.search(env)
- locale = env.get("preferences").as(Preferences).locale
- region = env.params.query["region"]?
+ prefs = env.get("preferences").as(Preferences)
+ locale = prefs.locale
- query = env.params.query["search_query"]?
- query ||= env.params.query["q"]?
+ region = env.params.query["region"]? || prefs.region
- if !query || query.empty?
+ query = Invidious::Search::Query.new(env.params.query, :regular, region)
+
+ if query.empty?
# Display the full page search box implemented in #1977
env.set "search", ""
templated "search_homepage", navbar_search: false
else
- page = env.params.query["page"]?.try &.to_i?
- page ||= 1
-
user = env.get? "user"
+ # An URL was copy/pasted in the search box.
+ # Redirect the user to the appropriate page.
+ if query.url?
+ return env.redirect UrlSanitizer.process(query.text).to_s
+ end
+
begin
- search_query, videos, operators = process_search_query(query, page, user, region: region)
+ if user
+ items = query.process(user.as(User))
+ else
+ items = query.process
+ end
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex
return error_template(500, ex)
end
- operator_hash = {} of String => String
- operators.each do |operator|
- key, value = operator.downcase.split(":")
- operator_hash[key] = value
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ # Pagination
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/search?#{query.to_http_params}",
+ current_page: query.page,
+ show_next: (items.size >= 20)
+ )
+
+ if query.type == Invidious::Search::Query::Type::Channel
+ env.set "search", "channel:#{query.channel} #{query.text}"
+ else
+ env.set "search", query.text
end
- env.set "search", query
templated "search"
end
end
+
+ def self.hashtag(env : HTTP::Server::Context)
+ locale = env.get("preferences").as(Preferences).locale
+
+ hashtag = env.params.url["hashtag"]?
+ if hashtag.nil? || hashtag.empty?
+ return error_template(400, "Invalid request")
+ end
+
+ page = env.params.query["page"]?
+ if page.nil?
+ page = 1
+ else
+ page = Math.max(1, page.to_i)
+ env.params.query.delete_all("page")
+ end
+
+ begin
+ items = Invidious::Hashtag.fetch(hashtag, page)
+ rescue ex
+ return error_template(500, ex)
+ end
+
+ # Pagination
+ hashtag_encoded = URI.encode_www_form(hashtag, space_to_plus: false)
+ page_nav_html = Frontend::Pagination.nav_numeric(locale,
+ base_url: "/hashtag/#{hashtag_encoded}",
+ current_page: page,
+ show_next: (items.size >= 60)
+ )
+
+ templated "hashtag"
+ end
end
diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr
index 7b1fa876..1de655d2 100644
--- a/src/invidious/routes/subscriptions.cr
+++ b/src/invidious/routes/subscriptions.cr
@@ -32,29 +32,16 @@ module Invidious::Routes::Subscriptions
end
end
- if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1
- action = "action_create_subscription_to_channel"
- elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1
- action = "action_remove_subscriptions"
- else
- return env.redirect referer
- end
-
channel_id = env.params.query["c"]?
channel_id ||= ""
- if !user.password
- # Sync subscriptions with YouTube
- subscribe_ajax(channel_id, action, env.request.headers)
- end
-
- case action
- when "action_create_subscription_to_channel"
+ case action = env.params.query["action"]?
+ when "create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
- when "action_remove_subscriptions"
+ when "remove_subscriptions"
Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else
return error_json(400, "Unsupported action #{action}")
@@ -82,14 +69,6 @@ module Invidious::Routes::Subscriptions
user = user.as(User)
sid = sid.as(String)
- if !user.password
- # Refresh account
- headers = HTTP::Headers.new
- headers["Cookie"] = env.request.headers["Cookie"]
-
- user, sid = get_user(sid, headers)
- end
-
action_takeout = env.params.query["action_takeout"]?.try &.to_i?
action_takeout ||= 0
action_takeout = action_takeout == 1
@@ -104,33 +83,8 @@ module Invidious::Routes::Subscriptions
if format == "json"
env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment"
- playlists = Invidious::Database::Playlists.select_like_iv(user.email)
- return JSON.build do |json|
- json.object do
- json.field "subscriptions", user.subscriptions
- json.field "watch_history", user.watched
- json.field "preferences", user.preferences
- json.field "playlists" do
- json.array do
- playlists.each do |playlist|
- json.object do
- json.field "title", playlist.title
- json.field "description", html_to_content(playlist.description_html)
- json.field "privacy", playlist.privacy.to_s
- json.field "videos" do
- json.array do
- Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id|
- json.string video_id
- end
- end
- end
- end
- end
- end
- end
- end
- end
+ return Invidious::User::Export.to_invidious(user)
else
env.response.content_type = "application/xml"
env.response.headers["content-disposition"] = "attachment"
diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index 3a92ef96..083087a9 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -21,7 +21,7 @@ module Invidious::Routes::VideoPlayback
end
# Sanity check, to avoid being used as an open proxy
- if !host.matches?(/[\w-]+.googlevideo.com/)
+ if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/)
return error_template(400, "Invalid \"host\" parameter.")
end
@@ -35,7 +35,15 @@ module Invidious::Routes::VideoPlayback
end
end
- client = make_client(URI.parse(host), region)
+ # See: https://github.com/iv-org/invidious/issues/3302
+ range_header = env.request.headers["Range"]?
+ sq = query_params["sq"]?
+ if range_header.nil? && sq.nil?
+ range_for_head = query_params["range"]? || "0-640"
+ headers["Range"] = "bytes=#{range_for_head}"
+ end
+
+ client = make_client(URI.parse(host), region, force_resolve: true)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
@@ -50,7 +58,7 @@ module Invidious::Routes::VideoPlayback
if new_host != host
host = new_host
client.close
- client = make_client(URI.parse(new_host), region)
+ client = make_client(URI.parse(new_host), region, force_resolve: true)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
@@ -64,15 +72,23 @@ module Invidious::Routes::VideoPlayback
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
- client = make_client(URI.parse(host), region)
+ client = make_client(URI.parse(host), region, force_resolve: true)
rescue ex
error = ex.message
end
end
+ # Remove the Range header added previously.
+ headers.delete("Range") if range_header.nil?
+
+ playback_statistics = get_playback_statistic()
+ playback_statistics["totalRequests"] += 1
+
if response.status_code >= 400
env.response.content_type = "text/plain"
haltf env, response.status_code
+ else
+ playback_statistics["successfulRequests"] += 1
end
if url.includes? "&file=seg.ts"
@@ -91,14 +107,8 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- location = URI.parse(location)
- location = "#{location.request_target}&host=#{location.host}"
-
- if region
- location += "®ion=#{region}"
- end
-
- return env.redirect location
+ url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
+ return env.redirect url
end
IO.copy(resp.body_io, env.response)
@@ -122,7 +132,7 @@ module Invidious::Routes::VideoPlayback
end
# TODO: Record bytes written so we can restart after a chunk fails
- while true
+ loop do
if !range_end && content_length
range_end = content_length
end
@@ -155,10 +165,13 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- location = URI.parse(location)
- location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
+ url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
- env.redirect location
+ if title = query_params["title"]?
+ url = "#{url}&title=#{URI.encode_www_form(title)}"
+ end
+
+ env.redirect url
break
end
@@ -187,7 +200,7 @@ module Invidious::Routes::VideoPlayback
break
else
client.close
- client = make_client(URI.parse(host), region)
+ client = make_client(URI.parse(host), region, force_resolve: true)
end
end
@@ -244,6 +257,11 @@ module Invidious::Routes::VideoPlayback
# YouTube /videoplayback links expire after 6 hours,
# so we have a mechanism here to redirect to the latest version
def self.latest_version(env)
+ if CONFIG.invidious_companion.present?
+ invidious_companion = CONFIG.invidious_companion.sample
+ return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
+ end
+
id = env.params.query["id"]?
itag = env.params.query["itag"]?.try &.to_i?
@@ -252,7 +270,7 @@ module Invidious::Routes::VideoPlayback
return error_template(400, "Invalid video ID")
end
- if itag.nil? || itag <= 0 || itag >= 1000
+ if !itag.nil? && (itag <= 0 || itag >= 1000)
return error_template(400, "Invalid itag")
end
@@ -265,9 +283,19 @@ module Invidious::Routes::VideoPlayback
return error_template(403, "Administrator has disabled this endpoint.")
end
- video = get_video(id, region: region)
+ begin
+ video = get_video(id, region: region)
+ rescue ex : NotFoundException
+ return error_template(404, ex)
+ rescue ex
+ return error_template(500, ex)
+ end
- fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
+ if itag.nil?
+ fmt = video.fmt_stream[-1]?
+ else
+ fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag }
+ end
url = fmt.try &.["url"]?.try &.as_s
if !url
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
index 867ffa6a..e777b3f1 100644
--- a/src/invidious/routes/watch.cr
+++ b/src/invidious/routes/watch.cr
@@ -30,14 +30,6 @@ module Invidious::Routes::Watch
return env.redirect "/"
end
- embed_link = "/embed/#{id}"
- if env.params.query.size > 1
- embed_params = HTTP::Params.parse(env.params.query.to_s)
- embed_params.delete_all("v")
- embed_link += "?"
- embed_link += embed_params.to_s
- end
-
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
continuation = process_continuation(env.params.query, plid, id)
@@ -61,8 +53,9 @@ module Invidious::Routes::Watch
begin
video = get_video(id, region: params.region)
- rescue ex : VideoRedirect
- return env.redirect env.request.resource.gsub(id, ex.video_id)
+ rescue ex : NotFoundException
+ LOGGER.error("get_video not found: #{id} : #{ex.message}")
+ return error_template(404, ex)
rescue ex
LOGGER.error("get_video: #{id} : #{ex.message}")
return error_template(500, ex)
@@ -75,11 +68,11 @@ module Invidious::Routes::Watch
end
env.params.query.delete_all("iv_load_policy")
- if watched && preferences.watch_history && !watched.includes? id
+ if watched && preferences.watch_history
Invidious::Database::Users.mark_watched(user.as(User), id)
end
- if notifications && notifications.includes? id
+ if CONFIG.enable_user_notifications && notifications && notifications.includes? id
Invidious::Database::Users.remove_notification(user.as(User), id)
env.get("user").as(User).notifications.delete(id)
notifications.delete(id)
@@ -94,31 +87,31 @@ module Invidious::Routes::Watch
if source == "youtube"
begin
- comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
rescue ex
if preferences.comments[1] == "reddit"
- comments, reddit_thread = fetch_reddit_comments(id)
- comment_html = template_reddit_comments(comments, locale)
+ comments, reddit_thread = Comments.fetch_reddit(id)
+ comment_html = Frontend::Comments.template_reddit(comments, locale)
- comment_html = fill_links(comment_html, "https", "www.reddit.com")
- comment_html = replace_links(comment_html)
+ comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = Comments.replace_links(comment_html)
end
end
elsif source == "reddit"
begin
- comments, reddit_thread = fetch_reddit_comments(id)
- comment_html = template_reddit_comments(comments, locale)
+ comments, reddit_thread = Comments.fetch_reddit(id)
+ comment_html = Frontend::Comments.template_reddit(comments, locale)
- comment_html = fill_links(comment_html, "https", "www.reddit.com")
- comment_html = replace_links(comment_html)
+ comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com")
+ comment_html = Comments.replace_links(comment_html)
rescue ex
if preferences.comments[1] == "youtube"
- comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
end
end
else
- comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
+ comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end
comment_html ||= ""
@@ -128,10 +121,12 @@ module Invidious::Routes::Watch
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
end
+ # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -197,6 +192,14 @@ module Invidious::Routes::Watch
captions: video.captions
)
+ if CONFIG.invidious_companion.present?
+ invidious_companion = CONFIG.invidious_companion.sample
+ env.response.headers["Content-Security-Policy"] =
+ env.response.headers["Content-Security-Policy"]
+ .gsub("media-src", "media-src #{invidious_companion.public_url}")
+ .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
+ end
+
templated "watch"
end
@@ -248,20 +251,10 @@ module Invidious::Routes::Watch
end
end
- if env.params.query["action_mark_watched"]?
- action = "action_mark_watched"
- elsif env.params.query["action_mark_unwatched"]?
- action = "action_mark_unwatched"
- else
- return env.redirect referer
- end
-
- case action
- when "action_mark_watched"
- if !user.watched.includes? id
- Invidious::Database::Users.mark_watched(user, id)
- end
- when "action_mark_unwatched"
+ case action = env.params.query["action"]?
+ when "mark_watched"
+ Invidious::Database::Users.mark_watched(user, id)
+ when "mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
else
return error_json(400, "Unsupported action #{action}")
@@ -284,6 +277,12 @@ module Invidious::Routes::Watch
return error_template(400, "Invalid clip ID") if response["error"]?
if video_id = response.dig?("endpoint", "watchEndpoint", "videoId")
+ if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
+ start_time, end_time, _ = parse_clip_parameters(params)
+ env.params.query["start"] = start_time.to_s if start_time != nil
+ env.params.query["end"] = end_time.to_s if end_time != nil
+ end
+
return env.redirect "/watch?v=#{video_id}&#{env.params.query}"
else
return error_template(404, "The requested clip doesn't exist")
@@ -294,6 +293,9 @@ module Invidious::Routes::Watch
if CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.")
end
+ if CONFIG.invidious_companion.present?
+ return error_template(403, "Downloads should be routed through Companion when present")
+ end
title = env.params.body["title"]? || ""
video_id = env.params.body["id"]? || ""
@@ -306,27 +308,27 @@ module Invidious::Routes::Watch
download_widget = JSON.parse(selection)
extension = download_widget["ext"].as_s
- filename = "#{video_id}-#{title}.#{extension}"
+ filename = "#{title}-#{video_id}.#{extension}"
- # Pass form parameters as URL parameters for the handlers of both
- # /latest_version and /api/v1/captions. This avoids an un-necessary
- # redirect and duplicated (and hazardous) sanity checks.
- env.params.query["id"] = video_id
- env.params.query["title"] = filename
-
- # Delete the useless ones
+ # Delete the now useless URL parameters
env.params.body.delete("id")
env.params.body.delete("title")
env.params.body.delete("download_widget")
+ # Pass form parameters as URL parameters for the handlers of both
+ # /latest_version and /api/v1/captions. This avoids an un-necessary
+ # redirect and duplicated (and hazardous) sanity checks.
if label = download_widget["label"]?
# URL params specific to /api/v1/captions/:id
- env.params.query["label"] = URI.encode_www_form(label.as_s, space_to_plus: false)
+ env.params.url["id"] = video_id
+ env.params.query["title"] = filename
+ env.params.query["label"] = URI.decode_www_form(label.as_s)
return Invidious::Routes::API::V1::Videos.captions(env)
- elsif itag = download_widget["itag"]?.try &.as_i
+ elsif itag = download_widget["itag"]?.try &.as_i.to_s
# URL params specific to /latest_version
- env.params.query["itag"] = itag.to_s
+ env.params.query["id"] = video_id
+ env.params.query["title"] = filename
env.params.query["local"] = "true"
return Invidious::Routes::VideoPlayback.latest_version(env)
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index bd72c577..46b71f1f 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -1,130 +1,319 @@
module Invidious::Routing
- {% for http_method in {"get", "post", "delete", "options", "patch", "put", "head"} %}
+ extend self
+
+ {% for http_method in {"get", "post", "delete", "options", "patch", "put"} %}
macro {{http_method.id}}(path, controller, method = :handle)
- {{http_method.id}} \{{ path }} do |env|
+ unless Kemal::Utils.path_starts_with_slash?(\{{path}})
+ raise Kemal::Exceptions::InvalidPathStartException.new({{http_method}}, \{{path}})
+ end
+
+ Kemal::RouteHandler::INSTANCE.add_route({{http_method.upcase}}, \{{path}}) do |env|
\{{ controller }}.\{{ method.id }}(env)
end
end
{% end %}
-end
-
-macro define_user_routes
- # User login/out
- Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page
- Invidious::Routing.post "/login", Invidious::Routes::Login, :login
- Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout
- Invidious::Routing.get "/Captcha", Invidious::Routes::Login, :captcha
-
- # User preferences
- Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show
- Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update
- Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme
- Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control
- Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control
-
- # User account management
- Invidious::Routing.get "/change_password", Invidious::Routes::Account, :get_change_password
- Invidious::Routing.post "/change_password", Invidious::Routes::Account, :post_change_password
- Invidious::Routing.get "/delete_account", Invidious::Routes::Account, :get_delete
- Invidious::Routing.post "/delete_account", Invidious::Routes::Account, :post_delete
- Invidious::Routing.get "/clear_watch_history", Invidious::Routes::Account, :get_clear_history
- Invidious::Routing.post "/clear_watch_history", Invidious::Routes::Account, :post_clear_history
- Invidious::Routing.get "/authorize_token", Invidious::Routes::Account, :get_authorize_token
- Invidious::Routing.post "/authorize_token", Invidious::Routes::Account, :post_authorize_token
- Invidious::Routing.get "/token_manager", Invidious::Routes::Account, :token_manager
- Invidious::Routing.post "/token_ajax", Invidious::Routes::Account, :token_ajax
-end
-
-macro define_v1_api_routes
- {{namespace = Invidious::Routes::API::V1}}
- # Videos
- Invidious::Routing.get "/api/v1/videos/:id", {{namespace}}::Videos, :videos
- Invidious::Routing.get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards
- Invidious::Routing.get "/api/v1/captions/:id", {{namespace}}::Videos, :captions
- Invidious::Routing.get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
- Invidious::Routing.get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
-
- # Feeds
- Invidious::Routing.get "/api/v1/trending", {{namespace}}::Feeds, :trending
- Invidious::Routing.get "/api/v1/popular", {{namespace}}::Feeds, :popular
-
- # Channels
- Invidious::Routing.get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
- {% for route in {"videos", "latest", "playlists", "community", "search"} %}
- Invidious::Routing.get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
- Invidious::Routing.get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
- {% end %}
-
- # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community
- Invidious::Routing.get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect
- Invidious::Routing.get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect
-
-
- # Search
- Invidious::Routing.get "/api/v1/search", {{namespace}}::Search, :search
- Invidious::Routing.get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions
-
- # Authenticated
-
- # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr
- #
- # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
- # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
-
- Invidious::Routing.get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
- Invidious::Routing.post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
-
- Invidious::Routing.get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed
-
- Invidious::Routing.get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions
- Invidious::Routing.post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel
- Invidious::Routing.delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel
-
-
- Invidious::Routing.get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists
- Invidious::Routing.post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist
- Invidious::Routing.patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute
- Invidious::Routing.delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist
-
-
- Invidious::Routing.post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist
- Invidious::Routing.delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist
-
- Invidious::Routing.get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens
- Invidious::Routing.post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token
- Invidious::Routing.post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token
-
- Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
- Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
-
- # Misc
- Invidious::Routing.get "/api/v1/stats", {{namespace}}::Misc, :stats
- Invidious::Routing.get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist
- Invidious::Routing.get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist
- Invidious::Routing.get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes
-end
-
-macro define_api_manifest_routes
- Invidious::Routing.get "/api/manifest/dash/id/:id", Invidious::Routes::API::Manifest, :get_dash_video_id
-
- Invidious::Routing.get "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :get_dash_video_playback
- Invidious::Routing.get "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :get_dash_video_playback_greedy
-
- Invidious::Routing.options "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :options_dash_video_playback
- Invidious::Routing.options "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :options_dash_video_playback
-
- Invidious::Routing.get "/api/manifest/hls_playlist/*", Invidious::Routes::API::Manifest, :get_hls_playlist
- Invidious::Routing.get "/api/manifest/hls_variant/*", Invidious::Routes::API::Manifest, :get_hls_variant
-end
-
-macro define_video_playback_routes
- Invidious::Routing.get "/videoplayback", Invidious::Routes::VideoPlayback, :get_video_playback
- Invidious::Routing.get "/videoplayback/*", Invidious::Routes::VideoPlayback, :get_video_playback_greedy
-
- Invidious::Routing.options "/videoplayback", Invidious::Routes::VideoPlayback, :options_video_playback
- Invidious::Routing.options "/videoplayback/*", Invidious::Routes::VideoPlayback, :options_video_playback
-
- Invidious::Routing.get "/latest_version", Invidious::Routes::VideoPlayback, :latest_version
+
+ def register_all
+ {% unless flag?(:api_only) %}
+ get "/", Routes::Misc, :home
+ get "/privacy", Routes::Misc, :privacy
+ get "/licenses", Routes::Misc, :licenses
+ get "/redirect", Routes::Misc, :cross_instance_redirect
+
+ self.register_channel_routes
+ self.register_watch_routes
+
+ self.register_iv_playlist_routes
+ self.register_yt_playlist_routes
+
+ self.register_search_routes
+
+ self.register_user_routes
+ self.register_feed_routes
+
+ # Support push notifications via PubSubHubbub
+ get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get
+ post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post
+
+ if CONFIG.enable_user_notifications
+ get "/modify_notifications", Routes::Notifications, :modify
+ end
+ {% end %}
+
+ self.register_image_routes
+ self.register_api_v1_routes
+ self.register_api_manifest_routes
+ self.register_video_playback_routes
+ end
+
+ # -------------------
+ # Invidious routes
+ # -------------------
+
+ def register_user_routes
+ # User login/out
+ get "/login", Routes::Login, :login_page
+ post "/login", Routes::Login, :login
+ post "/signout", Routes::Login, :signout
+
+ # User preferences
+ get "/preferences", Routes::PreferencesRoute, :show
+ post "/preferences", Routes::PreferencesRoute, :update
+ get "/toggle_theme", Routes::PreferencesRoute, :toggle_theme
+ get "/data_control", Routes::PreferencesRoute, :data_control
+ post "/data_control", Routes::PreferencesRoute, :update_data_control
+
+ # User account management
+ get "/change_password", Routes::Account, :get_change_password
+ post "/change_password", Routes::Account, :post_change_password
+ get "/delete_account", Routes::Account, :get_delete
+ post "/delete_account", Routes::Account, :post_delete
+ get "/clear_watch_history", Routes::Account, :get_clear_history
+ post "/clear_watch_history", Routes::Account, :post_clear_history
+ get "/authorize_token", Routes::Account, :get_authorize_token
+ post "/authorize_token", Routes::Account, :post_authorize_token
+ get "/token_manager", Routes::Account, :token_manager
+ post "/token_ajax", Routes::Account, :token_ajax
+ post "/subscription_ajax", Routes::Subscriptions, :toggle_subscription
+ get "/subscription_manager", Routes::Subscriptions, :subscription_manager
+ end
+
+ def register_iv_playlist_routes
+ get "/create_playlist", Routes::Playlists, :new
+ post "/create_playlist", Routes::Playlists, :create
+ get "/subscribe_playlist", Routes::Playlists, :subscribe
+ get "/delete_playlist", Routes::Playlists, :delete_page
+ post "/delete_playlist", Routes::Playlists, :delete
+ get "/edit_playlist", Routes::Playlists, :edit
+ post "/edit_playlist", Routes::Playlists, :update
+ get "/add_playlist_items", Routes::Playlists, :add_playlist_items_page
+ post "/playlist_ajax", Routes::Playlists, :playlist_ajax
+ end
+
+ def register_feed_routes
+ # Feeds
+ get "/view_all_playlists", Routes::Feeds, :view_all_playlists_redirect
+ get "/feed/playlists", Routes::Feeds, :playlists
+ get "/feed/popular", Routes::Feeds, :popular
+ get "/feed/trending", Routes::Feeds, :trending
+ get "/feed/subscriptions", Routes::Feeds, :subscriptions
+ get "/feed/history", Routes::Feeds, :history
+
+ # RSS Feeds
+ get "/feed/channel/:ucid", Routes::Feeds, :rss_channel
+ get "/feed/private", Routes::Feeds, :rss_private
+ get "/feed/playlist/:plid", Routes::Feeds, :rss_playlist
+ get "/feeds/videos.xml", Routes::Feeds, :rss_videos
+ end
+
+ # -------------------
+ # Youtube routes
+ # -------------------
+
+ def register_channel_routes
+ get "/channel/:ucid", Routes::Channels, :home
+ get "/channel/:ucid/home", Routes::Channels, :home
+ get "/channel/:ucid/videos", Routes::Channels, :videos
+ get "/channel/:ucid/shorts", Routes::Channels, :shorts
+ get "/channel/:ucid/streams", Routes::Channels, :streams
+ get "/channel/:ucid/podcasts", Routes::Channels, :podcasts
+ get "/channel/:ucid/releases", Routes::Channels, :releases
+ get "/channel/:ucid/courses", Routes::Channels, :courses
+ get "/channel/:ucid/playlists", Routes::Channels, :playlists
+ get "/channel/:ucid/community", Routes::Channels, :community
+ get "/channel/:ucid/posts", Routes::Channels, :community
+ get "/channel/:ucid/channels", Routes::Channels, :channels
+ get "/channel/:ucid/about", Routes::Channels, :about
+
+ get "/channel/:ucid/live", Routes::Channels, :live
+ get "/user/:user/live", Routes::Channels, :live
+ get "/c/:user/live", Routes::Channels, :live
+ get "/post/:id", Routes::Channels, :post
+
+ # Channel catch-all, to redirect future routes to the channel's home
+ # NOTE: defined last in order to be processed after the other routes
+ get "/channel/:ucid/*", Routes::Channels, :redirect_home
+
+ # /c/LinusTechTips
+ get "/c/:user", Routes::Channels, :brand_redirect
+ get "/c/:user/:tab", Routes::Channels, :brand_redirect
+
+ # /user/linustechtips (Not always the same as /c/)
+ get "/user/:user", Routes::Channels, :brand_redirect
+ get "/user/:user/:tab", Routes::Channels, :brand_redirect
+
+ # /@LinusTechTips (Handle)
+ get "/@:user", Routes::Channels, :brand_redirect
+ get "/@:user/:tab", Routes::Channels, :brand_redirect
+
+ # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow
+ get "/attribution_link", Routes::Channels, :brand_redirect
+ get "/attribution_link/:tab", Routes::Channels, :brand_redirect
+
+ # /profile?user=linustechtips
+ get "/profile", Routes::Channels, :profile
+ get "/profile/*", Routes::Channels, :profile
+ end
+
+ def register_watch_routes
+ get "/watch", Routes::Watch, :handle
+ post "/watch_ajax", Routes::Watch, :mark_watched
+ get "/watch/:id", Routes::Watch, :redirect
+ get "/live/:id", Routes::Watch, :redirect
+ get "/shorts/:id", Routes::Watch, :redirect
+ get "/clip/:clip", Routes::Watch, :clip
+ get "/w/:id", Routes::Watch, :redirect
+ get "/v/:id", Routes::Watch, :redirect
+ get "/e/:id", Routes::Watch, :redirect
+
+ post "/download", Routes::Watch, :download
+
+ get "/embed/", Routes::Embed, :redirect
+ get "/embed/:id", Routes::Embed, :show
+ end
+
+ def register_yt_playlist_routes
+ get "/playlist", Routes::Playlists, :show
+ get "/mix", Routes::Playlists, :mix
+ get "/watch_videos", Routes::Playlists, :watch_videos
+ end
+
+ def register_search_routes
+ get "/opensearch.xml", Routes::Search, :opensearch
+ get "/results", Routes::Search, :results
+ get "/search", Routes::Search, :search
+ get "/hashtag/:hashtag", Routes::Search, :hashtag
+ end
+
+ # -------------------
+ # Media proxy routes
+ # -------------------
+
+ def register_api_manifest_routes
+ get "/api/manifest/dash/id/:id", Routes::API::Manifest, :get_dash_video_id
+
+ get "/api/manifest/dash/id/videoplayback", Routes::API::Manifest, :get_dash_video_playback
+ get "/api/manifest/dash/id/videoplayback/*", Routes::API::Manifest, :get_dash_video_playback_greedy
+
+ options "/api/manifest/dash/id/videoplayback", Routes::API::Manifest, :options_dash_video_playback
+ options "/api/manifest/dash/id/videoplayback/*", Routes::API::Manifest, :options_dash_video_playback
+
+ get "/api/manifest/hls_playlist/*", Routes::API::Manifest, :get_hls_playlist
+ get "/api/manifest/hls_variant/*", Routes::API::Manifest, :get_hls_variant
+ end
+
+ def register_video_playback_routes
+ get "/videoplayback", Routes::VideoPlayback, :get_video_playback
+ get "/videoplayback/*", Routes::VideoPlayback, :get_video_playback_greedy
+
+ options "/videoplayback", Routes::VideoPlayback, :options_video_playback
+ options "/videoplayback/*", Routes::VideoPlayback, :options_video_playback
+
+ get "/latest_version", Routes::VideoPlayback, :latest_version
+ end
+
+ def register_image_routes
+ get "/ggpht/*", Routes::Images, :ggpht
+ options "/sb/:authority/:id/:storyboard/:index", Routes::Images, :options_storyboard
+ get "/sb/:authority/:id/:storyboard/:index", Routes::Images, :get_storyboard
+ get "/s_p/:id/:name", Routes::Images, :s_p_image
+ get "/yts/img/:name", Routes::Images, :yts_image
+ get "/vi/:id/:name", Routes::Images, :thumbnails
+ end
+
+ # -------------------
+ # API routes
+ # -------------------
+
+ def register_api_v1_routes
+ {% begin %}
+ {{namespace = Routes::API::V1}}
+
+ # Videos
+ get "/api/v1/videos/:id", {{namespace}}::Videos, :videos
+ get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards
+ get "/api/v1/captions/:id", {{namespace}}::Videos, :captions
+ get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
+ get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
+ get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
+ get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts
+
+ # Feeds
+ get "/api/v1/trending", {{namespace}}::Feeds, :trending
+ get "/api/v1/popular", {{namespace}}::Feeds, :popular
+
+ # Channels
+ get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
+ get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest
+ get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos
+ get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
+ get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
+ get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
+ get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
+ get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses
+ get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
+ get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
+ get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community
+ get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
+ get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search
+
+ # Posts
+ get "/api/v1/post/:id", {{namespace}}::Channels, :post
+ get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments
+
+ # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community
+ get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect
+ get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect
+
+ # Search
+ get "/api/v1/search", {{namespace}}::Search, :search
+ get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions
+ get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag
+
+
+ # Authenticated
+
+ get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
+ post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
+
+ get "/api/v1/auth/export/invidious", {{namespace}}::Authenticated, :export_invidious
+ post "/api/v1/auth/import/invidious", {{namespace}}::Authenticated, :import_invidious
+
+ get "/api/v1/auth/history", {{namespace}}::Authenticated, :get_history
+ post "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_watched
+ delete "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_unwatched
+ delete "/api/v1/auth/history", {{namespace}}::Authenticated, :clear_history
+
+ get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed
+
+ get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions
+ post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel
+ delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel
+
+ get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists
+ post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist
+ patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute
+ delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist
+ post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist
+ delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist
+
+ get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens
+ post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token
+ post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token
+
+ if CONFIG.enable_user_notifications
+ get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ end
+
+ # Misc
+ get "/api/v1/stats", {{namespace}}::Misc, :stats
+ get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist
+ get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist
+ get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes
+ get "/api/v1/resolveurl", {{namespace}}::Misc, :resolve_url
+ {% end %}
+ end
end
diff --git a/src/invidious/search.cr b/src/invidious/search.cr
deleted file mode 100644
index ae106bf6..00000000
--- a/src/invidious/search.cr
+++ /dev/null
@@ -1,254 +0,0 @@
-class ChannelSearchException < InfoException
- getter channel : String
-
- def initialize(@channel)
- end
-end
-
-def channel_search(query, page, channel) : Array(SearchItem)
- response = YT_POOL.client &.get("/channel/#{channel}")
-
- if response.status_code == 404
- response = YT_POOL.client &.get("/user/#{channel}")
- response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404
- initial_data = extract_initial_data(response.body)
- ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
- raise ChannelSearchException.new(channel) if !ucid
- else
- ucid = channel
- end
-
- continuation = produce_channel_search_continuation(ucid, query, page)
- response_json = YoutubeAPI.browse(continuation)
-
- continuation_items = response_json["onResponseReceivedActions"]?
- .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
- return [] of SearchItem if !continuation_items
-
- items = [] of SearchItem
- continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
- extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
- end
-
- return items
-end
-
-def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem)
- return [] of SearchItem if query.empty?
-
- client_config = YoutubeAPI::ClientConfig.new(region: region)
- initial_data = YoutubeAPI.search(query, search_params, client_config: client_config)
-
- return extract_items(initial_data)
-end
-
-def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "",
- duration : String = "", features : Array(String) = [] of String)
- object = {
- "1:varint" => 0_i64,
- "2:embedded" => {} of String => Int64,
- "9:varint" => ((page - 1) * 20).to_i64,
- }
-
- case sort
- when "relevance"
- object["1:varint"] = 0_i64
- when "rating"
- object["1:varint"] = 1_i64
- when "upload_date", "date"
- object["1:varint"] = 2_i64
- when "view_count", "views"
- object["1:varint"] = 3_i64
- else
- raise "No sort #{sort}"
- end
-
- case date
- when "hour"
- object["2:embedded"].as(Hash)["1:varint"] = 1_i64
- when "today"
- object["2:embedded"].as(Hash)["1:varint"] = 2_i64
- when "week"
- object["2:embedded"].as(Hash)["1:varint"] = 3_i64
- when "month"
- object["2:embedded"].as(Hash)["1:varint"] = 4_i64
- when "year"
- object["2:embedded"].as(Hash)["1:varint"] = 5_i64
- else nil # Ignore
- end
-
- case content_type
- when "video"
- object["2:embedded"].as(Hash)["2:varint"] = 1_i64
- when "channel"
- object["2:embedded"].as(Hash)["2:varint"] = 2_i64
- when "playlist"
- object["2:embedded"].as(Hash)["2:varint"] = 3_i64
- when "movie"
- object["2:embedded"].as(Hash)["2:varint"] = 4_i64
- when "show"
- object["2:embedded"].as(Hash)["2:varint"] = 5_i64
- when "all"
- #
- else
- object["2:embedded"].as(Hash)["2:varint"] = 1_i64
- end
-
- case duration
- when "short"
- object["2:embedded"].as(Hash)["3:varint"] = 1_i64
- when "long"
- object["2:embedded"].as(Hash)["3:varint"] = 2_i64
- else nil # Ignore
- end
-
- features.each do |feature|
- case feature
- when "hd"
- object["2:embedded"].as(Hash)["4:varint"] = 1_i64
- when "subtitles"
- object["2:embedded"].as(Hash)["5:varint"] = 1_i64
- when "creative_commons", "cc"
- object["2:embedded"].as(Hash)["6:varint"] = 1_i64
- when "3d"
- object["2:embedded"].as(Hash)["7:varint"] = 1_i64
- when "live", "livestream"
- object["2:embedded"].as(Hash)["8:varint"] = 1_i64
- when "purchased"
- object["2:embedded"].as(Hash)["9:varint"] = 1_i64
- when "4k"
- object["2:embedded"].as(Hash)["14:varint"] = 1_i64
- when "360"
- object["2:embedded"].as(Hash)["15:varint"] = 1_i64
- when "location"
- object["2:embedded"].as(Hash)["23:varint"] = 1_i64
- when "hdr"
- object["2:embedded"].as(Hash)["25:varint"] = 1_i64
- else nil # Ignore
- end
- end
-
- if object["2:embedded"].as(Hash).empty?
- object.delete("2:embedded")
- end
-
- params = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- return params
-end
-
-def produce_channel_search_continuation(ucid, query, page)
- if page <= 1
- idx = 0_i64
- else
- idx = 30_i64 * (page - 1)
- end
-
- object = {
- "80226972:embedded" => {
- "2:string" => ucid,
- "3:base64" => {
- "2:string" => "search",
- "6:varint" => 1_i64,
- "7:varint" => 1_i64,
- "12:varint" => 1_i64,
- "15:base64" => {
- "3:varint" => idx,
- },
- "23:varint" => 0_i64,
- },
- "11:string" => query,
- "35:string" => "browse-feed#{ucid}search",
- },
- }
-
- continuation = object.try { |i| Protodec::Any.cast_json(i) }
- .try { |i| Protodec::Any.from_json(i) }
- .try { |i| Base64.urlsafe_encode(i) }
- .try { |i| URI.encode_www_form(i) }
-
- return continuation
-end
-
-def process_search_query(query, page, user, region)
- if user
- user = user.as(Invidious::User)
- view_name = "subscriptions_#{sha256(user.email)}"
- end
-
- channel = nil
- content_type = "all"
- date = ""
- duration = ""
- features = [] of String
- sort = "relevance"
- subscriptions = nil
-
- operators = query.split(" ").select(&.match(/\w+:[\w,]+/))
- operators.each do |operator|
- key, value = operator.downcase.split(":")
-
- case key
- when "channel", "user"
- channel = operator.split(":")[-1]
- when "content_type", "type"
- content_type = value
- when "date"
- date = value
- when "duration"
- duration = value
- when "feature", "features"
- features = value.split(",")
- when "sort"
- sort = value
- when "subscriptions"
- subscriptions = value == "true"
- else
- operators.delete(operator)
- end
- end
-
- search_query = (query.split(" ") - operators).join(" ")
-
- if channel
- items = channel_search(search_query, page, channel)
- elsif subscriptions
- if view_name
- items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM (
- SELECT *,
- to_tsvector(#{view_name}.title) ||
- to_tsvector(#{view_name}.author)
- as document
- FROM #{view_name}
- ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo)
- else
- items = [] of ChannelVideo
- end
- else
- search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type,
- duration: duration, features: features)
-
- items = search(search_query, search_params, region)
- end
-
- # Light processing to flatten search results out of Categories.
- # They should ideally be supported in the future.
- items_without_category = [] of SearchItem | ChannelVideo
- items.each do |i|
- if i.is_a? Category
- i.contents.each do |nest_i|
- if !nest_i.is_a? Video
- items_without_category << nest_i
- end
- end
- else
- items_without_category << i
- end
- end
-
- {search_query, items_without_category, operators}
-end
diff --git a/src/invidious/search/ctoken.cr b/src/invidious/search/ctoken.cr
new file mode 100644
index 00000000..161065e0
--- /dev/null
+++ b/src/invidious/search/ctoken.cr
@@ -0,0 +1,32 @@
+def produce_channel_search_continuation(ucid, query, page)
+ if page <= 1
+ idx = 0_i64
+ else
+ idx = 30_i64 * (page - 1)
+ end
+
+ object = {
+ "80226972:embedded" => {
+ "2:string" => ucid,
+ "3:base64" => {
+ "2:string" => "search",
+ "6:varint" => 1_i64,
+ "7:varint" => 1_i64,
+ "12:varint" => 1_i64,
+ "15:base64" => {
+ "3:varint" => idx,
+ },
+ "23:varint" => 0_i64,
+ },
+ "11:string" => query,
+ "35:string" => "browse-feed#{ucid}search",
+ },
+ }
+
+ continuation = object.try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+
+ return continuation
+end
diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr
new file mode 100644
index 00000000..bc2715cf
--- /dev/null
+++ b/src/invidious/search/filters.cr
@@ -0,0 +1,376 @@
+require "protodec/utils"
+require "http/params"
+
+module Invidious::Search
+ struct Filters
+ # Values correspond to { "2:embedded": { "1:varint": }}
+ # except for "None" which is only used by us (= nothing selected)
+ enum Date
+ None = 0
+ Hour = 1
+ Today = 2
+ Week = 3
+ Month = 4
+ Year = 5
+ end
+
+ # Values correspond to { "2:embedded": { "2:varint": }}
+ # except for "All" which is only used by us (= nothing selected)
+ enum Type
+ All = 0
+ Video = 1
+ Channel = 2
+ Playlist = 3
+ Movie = 4
+
+ # Has it been removed?
+ # (Not available on youtube's UI)
+ Show = 5
+ end
+
+ # Values correspond to { "2:embedded": { "3:varint": }}
+ # except for "None" which is only used by us (= nothing selected)
+ enum Duration
+ None = 0
+ Short = 1 # "Under 4 minutes"
+ Long = 2 # "Over 20 minutes"
+ Medium = 3 # "4 - 20 minutes"
+ end
+
+ # Note: flag enums automatically generate
+ # "none" and "all" members
+ @[Flags]
+ enum Features
+ Live
+ FourK # "4K"
+ HD
+ Subtitles # "Subtitles/CC"
+ CCommons # "Creative Commons"
+ ThreeSixty # "360°"
+ VR180
+ ThreeD # "3D"
+ HDR
+ Location
+ Purchased
+ end
+
+ # Values correspond to { "1:varint": }
+ enum Sort
+ Relevance = 0
+ Rating = 1
+ Date = 2
+ Views = 3
+ end
+
+ # Parameters are sorted as on Youtube
+ property date : Date
+ property type : Type
+ property duration : Duration
+ property features : Features
+ property sort : Sort
+
+ def initialize(
+ *, # All parameters must be named
+ @date : Date = Date::None,
+ @type : Type = Type::All,
+ @duration : Duration = Duration::None,
+ @features : Features = Features::None,
+ @sort : Sort = Sort::Relevance,
+ )
+ end
+
+ def default? : Bool
+ return @date.none? && @type.all? && @duration.none? && \
+ @features.none? && @sort.relevance?
+ end
+
+ # -------------------
+ # Invidious params
+ # -------------------
+
+ def self.parse_features(raw : Array(String)) : Features
+ # Initialize return variable
+ features = Features.new(0)
+
+ raw.each do |ft|
+ case ft.downcase
+ when "live", "livestream"
+ features = features | Features::Live
+ when "4k" then features = features | Features::FourK
+ when "hd" then features = features | Features::HD
+ when "subtitles" then features = features | Features::Subtitles
+ when "creative_commons", "commons", "cc"
+ features = features | Features::CCommons
+ when "360" then features = features | Features::ThreeSixty
+ when "vr180" then features = features | Features::VR180
+ when "3d" then features = features | Features::ThreeD
+ when "hdr" then features = features | Features::HDR
+ when "location" then features = features | Features::Location
+ when "purchased" then features = features | Features::Purchased
+ end
+ end
+
+ return features
+ end
+
+ def self.format_features(features : Features) : String
+ # Directly return an empty string if there are no features
+ return "" if features.none?
+
+ # Initialize return variable
+ str = [] of String
+
+ str << "live" if features.live?
+ str << "4k" if features.four_k?
+ str << "hd" if features.hd?
+ str << "subtitles" if features.subtitles?
+ str << "commons" if features.c_commons?
+ str << "360" if features.three_sixty?
+ str << "vr180" if features.vr180?
+ str << "3d" if features.three_d?
+ str << "hdr" if features.hdr?
+ str << "location" if features.location?
+ str << "purchased" if features.purchased?
+
+ return str.join(',')
+ end
+
+ def self.from_legacy_filters(str : String) : {Filters, String, String, Bool}
+ # Split search query on spaces
+ members = str.split(' ')
+
+ # Output variables
+ channel = ""
+ filters = Filters.new
+ subscriptions = false
+
+ # Array to hold the non-filter members
+ query = [] of String
+
+ # Parse!
+ members.each do |substr|
+ # Separator operators
+ operators = substr.split(':')
+
+ case operators[0]
+ when "user", "channel"
+ next if operators.size != 2
+ channel = operators[1]
+ #
+ when "type", "content_type"
+ next if operators.size != 2
+ type = Type.parse?(operators[1])
+ filters.type = type if !type.nil?
+ #
+ when "date"
+ next if operators.size != 2
+ date = Date.parse?(operators[1])
+ filters.date = date if !date.nil?
+ #
+ when "duration"
+ next if operators.size != 2
+ duration = Duration.parse?(operators[1])
+ filters.duration = duration if !duration.nil?
+ #
+ when "feature", "features"
+ next if operators.size != 2
+ features = parse_features(operators[1].split(','))
+ filters.features = features if !features.nil?
+ #
+ when "sort"
+ next if operators.size != 2
+ sort = Sort.parse?(operators[1])
+ filters.sort = sort if !sort.nil?
+ #
+ when "subscriptions"
+ next if operators.size != 2
+ subscriptions = {"true", "on", "yes", "1"}.any?(&.== operators[1])
+ #
+ else
+ query << substr
+ end
+ end
+
+ # Re-assemble query (without filters)
+ cleaned_query = query.join(' ')
+
+ return {filters, channel, cleaned_query, subscriptions}
+ end
+
+ def self.from_iv_params(params : HTTP::Params) : Filters
+ # Temporary variables
+ filters = Filters.new
+
+ if type = params["type"]?
+ filters.type = Type.parse?(type) || Type::All
+ params.delete("type")
+ end
+
+ if date = params["date"]?
+ filters.date = Date.parse?(date) || Date::None
+ params.delete("date")
+ end
+
+ if duration = params["duration"]?
+ filters.duration = Duration.parse?(duration) || Duration::None
+ params.delete("duration")
+ end
+
+ features = params.fetch_all("features")
+ if !features.empty?
+ # Un-array input so it can be treated as a comma-separated list
+ features = features[0].split(',') if features.size == 1
+
+ filters.features = parse_features(features) || Features::None
+ params.delete_all("features")
+ end
+
+ if sort = params["sort"]?
+ filters.sort = Sort.parse?(sort) || Sort::Relevance
+ params.delete("sort")
+ end
+
+ return filters
+ end
+
+ def to_iv_params : HTTP::Params
+ # Temporary variables
+ raw_params = {} of String => Array(String)
+
+ raw_params["date"] = [@date.to_s.underscore] if !@date.none?
+ raw_params["type"] = [@type.to_s.underscore] if !@type.all?
+ raw_params["sort"] = [@sort.to_s.underscore] if !@sort.relevance?
+
+ if !@duration.none?
+ raw_params["duration"] = [@duration.to_s.underscore]
+ end
+
+ if !@features.none?
+ raw_params["features"] = [Filters.format_features(@features)]
+ end
+
+ return HTTP::Params.new(raw_params)
+ end
+
+ # -------------------
+ # Youtube params
+ # -------------------
+
+ # Produce the youtube search parameters for the
+ # innertube API (base64-encoded protobuf object).
+ def to_yt_params(page : Int = 1) : String
+ # Initialize the embedded protobuf object
+ embedded = {} of String => Int64
+
+ # Add these field only if associated parameter is selected
+ embedded["1:varint"] = @date.to_i64 if !@date.none?
+ embedded["2:varint"] = @type.to_i64 if !@type.all?
+ embedded["3:varint"] = @duration.to_i64 if !@duration.none?
+
+ if !@features.none?
+ # All features have a value of "1" when enabled, and
+ # the field is omitted when the feature is no selected.
+ embedded["4:varint"] = 1_i64 if @features.includes?(Features::HD)
+ embedded["5:varint"] = 1_i64 if @features.includes?(Features::Subtitles)
+ embedded["6:varint"] = 1_i64 if @features.includes?(Features::CCommons)
+ embedded["7:varint"] = 1_i64 if @features.includes?(Features::ThreeD)
+ embedded["8:varint"] = 1_i64 if @features.includes?(Features::Live)
+ embedded["9:varint"] = 1_i64 if @features.includes?(Features::Purchased)
+ embedded["14:varint"] = 1_i64 if @features.includes?(Features::FourK)
+ embedded["15:varint"] = 1_i64 if @features.includes?(Features::ThreeSixty)
+ embedded["23:varint"] = 1_i64 if @features.includes?(Features::Location)
+ embedded["25:varint"] = 1_i64 if @features.includes?(Features::HDR)
+ embedded["26:varint"] = 1_i64 if @features.includes?(Features::VR180)
+ end
+
+ # Initialize an empty protobuf object
+ object = {} of String => (Int64 | String | Hash(String, Int64))
+
+ # As usual, everything can be omitted if it has no value
+ object["2:embedded"] = embedded if !embedded.empty?
+
+ # Default sort is "relevance", so when this option is selected,
+ # the associated field can be omitted.
+ if !@sort.relevance?
+ object["1:varint"] = @sort.to_i64
+ end
+
+ # Add page number (if provided)
+ if page > 1
+ object["9:varint"] = ((page - 1) * 20).to_i64
+ end
+
+ # Prevent censoring of self harm topics
+ # See https://github.com/iv-org/invidious/issues/4398
+ object["30:varint"] = 1.to_i64
+
+ return object
+ .try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+ end
+
+ # Function to parse the `sp` URL parameter from Youtube
+ # search page. It's a base64-encoded protobuf object.
+ def self.from_yt_params(params : HTTP::Params) : Filters
+ # Initialize output variable
+ filters = Filters.new
+
+ # Get parameter, and check emptyness
+ search_params = params["sp"]?
+
+ if search_params.nil? || search_params.empty?
+ return filters
+ end
+
+ # Decode protobuf object
+ object = search_params
+ .try { |i| URI.decode_www_form(i) }
+ .try { |i| Base64.decode(i) }
+ .try { |i| IO::Memory.new(i) }
+ .try { |i| Protodec::Any.parse(i) }
+
+ # Parse items from embedded object
+ if embedded = object["2:0:embedded"]?
+ # All the following fields (date, type, duration) are optional.
+ if date = embedded["1:0:varint"]?
+ filters.date = Date.from_value?(date.as_i) || Date::None
+ end
+
+ if type = embedded["2:0:varint"]?
+ filters.type = Type.from_value?(type.as_i) || Type::All
+ end
+
+ if duration = embedded["3:0:varint"]?
+ filters.duration = Duration.from_value?(duration.as_i) || Duration::None
+ end
+
+ # All features should have a value of "1" when enabled, and
+ # the field should be omitted when the feature is no selected.
+ features = 0
+ features += (embedded["4:0:varint"]?.try &.as_i == 1_i64) ? Features::HD.value : 0
+ features += (embedded["5:0:varint"]?.try &.as_i == 1_i64) ? Features::Subtitles.value : 0
+ features += (embedded["6:0:varint"]?.try &.as_i == 1_i64) ? Features::CCommons.value : 0
+ features += (embedded["7:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeD.value : 0
+ features += (embedded["8:0:varint"]?.try &.as_i == 1_i64) ? Features::Live.value : 0
+ features += (embedded["9:0:varint"]?.try &.as_i == 1_i64) ? Features::Purchased.value : 0
+ features += (embedded["14:0:varint"]?.try &.as_i == 1_i64) ? Features::FourK.value : 0
+ features += (embedded["15:0:varint"]?.try &.as_i == 1_i64) ? Features::ThreeSixty.value : 0
+ features += (embedded["23:0:varint"]?.try &.as_i == 1_i64) ? Features::Location.value : 0
+ features += (embedded["25:0:varint"]?.try &.as_i == 1_i64) ? Features::HDR.value : 0
+ features += (embedded["26:0:varint"]?.try &.as_i == 1_i64) ? Features::VR180.value : 0
+
+ filters.features = Features.from_value?(features) || Features::None
+ end
+
+ if sort = object["1:0:varint"]?
+ filters.sort = Sort.from_value?(sort.as_i) || Sort::Relevance
+ end
+
+ # Remove URL parameter and return result
+ params.delete("sp")
+ return filters
+ end
+ end
+end
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
new file mode 100644
index 00000000..25edb936
--- /dev/null
+++ b/src/invidious/search/processors.cr
@@ -0,0 +1,56 @@
+module Invidious::Search
+ module Processors
+ extend self
+
+ # Regular search (`/search` endpoint)
+ def regular(query : Query) : Array(SearchItem)
+ search_params = query.filters.to_yt_params(page: query.page)
+
+ client_config = YoutubeAPI::ClientConfig.new(region: query.region)
+ initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
+
+ items, _ = extract_items(initial_data)
+ return items.reject!(Category)
+ end
+
+ # Search a youtube channel
+ # TODO: clean code, and rely more on YoutubeAPI
+ def channel(query : Query) : Array(SearchItem)
+ response = YT_POOL.client &.get("/channel/#{query.channel}")
+
+ if response.status_code == 404
+ response = YT_POOL.client &.get("/user/#{query.channel}")
+ response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
+ initial_data = extract_initial_data(response.body)
+ ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
+ raise ChannelSearchException.new(query.channel) if !ucid
+ else
+ ucid = query.channel
+ end
+
+ continuation = produce_channel_search_continuation(ucid, query.text, query.page)
+ response_json = YoutubeAPI.browse(continuation)
+
+ items, _ = extract_items(response_json, "", ucid)
+ return items.reject!(Category)
+ end
+
+ # Search inside of user subscriptions
+ def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
+ view_name = "subscriptions_#{sha256(user.email)}"
+
+ return PG_DB.query_all("
+ SELECT id,title,published,updated,ucid,author,length_seconds
+ FROM (
+ SELECT *,
+ to_tsvector(#{view_name}.title) ||
+ to_tsvector(#{view_name}.author)
+ as document
+ FROM #{view_name}
+ ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
+ query.text, (query.page - 1) * 20,
+ as: ChannelVideo
+ )
+ end
+ end
+end
diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr
new file mode 100644
index 00000000..94a92e23
--- /dev/null
+++ b/src/invidious/search/query.cr
@@ -0,0 +1,168 @@
+module Invidious::Search
+ class Query
+ enum Type
+ # Types related to YouTube
+ Regular # Youtube search page
+ Channel # Youtube channel search box
+
+ # Types specific to Invidious
+ Subscriptions # Search user subscriptions
+ Playlist # "Add playlist item" search
+ end
+
+ getter type : Type = Type::Regular
+
+ @raw_query : String
+ @query : String = ""
+
+ property filters : Filters = Filters.new
+ property page : Int32
+ property region : String?
+ property channel : String = ""
+
+ # Flag that indicates if the smart search features have been disabled.
+ @inhibit_ssf : Bool = false
+
+ # Return true if @raw_query is either `nil` or empty
+ private def empty_raw_query?
+ return @raw_query.empty?
+ end
+
+ # Same as `empty_raw_query?`, but named for external use
+ def empty?
+ return self.empty_raw_query?
+ end
+
+ # Getter for the query string.
+ # It is named `text` to reduce confusion (`search_query.text` makes more
+ # sense than `search_query.query`)
+ def text
+ return @query
+ end
+
+ # Initialize a new search query.
+ # Parameters are used to get the query string, the page number
+ # and the search filters (if any). Type tells this function
+ # where it is being called from (See `Type` above).
+ def initialize(
+ params : HTTP::Params,
+ @type : Type = Type::Regular,
+ @region : String? = nil,
+ )
+ # Get the raw search query string (common to all search types). In
+ # Regular search mode, also look for the `search_query` URL parameter
+ _raw_query = params["q"]?
+ _raw_query ||= params["search_query"]? if @type.regular?
+ _raw_query ||= ""
+
+ # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs.
+ @raw_query = _raw_query.strip
+
+ # Check for smart features (ex: URL search) inhibitor (backslash).
+ # If inhibitor is present, remove it.
+ if @raw_query.starts_with?('\\')
+ @inhibit_ssf = true
+ @raw_query = @raw_query[1..]
+ end
+
+ # Get the page number (also common to all search types)
+ @page = params["page"]?.try &.to_i? || 1
+
+ # Stop here if raw query is empty
+ # NOTE: maybe raise in the future?
+ return if self.empty_raw_query?
+
+ # Specific handling
+ case @type
+ when .channel?
+ # In "channel search" mode, filters are ignored, but we still parse
+ # the query prevent transmission of legacy filters to youtube.
+ #
+ _, _, @query, _ = Filters.from_legacy_filters(@raw_query)
+ #
+ when .playlist?
+ # In "add playlist item" mode, filters are parsed from the query
+ # string itself (legacy), and the channel is ignored.
+ #
+ @filters, _, @query, _ = Filters.from_legacy_filters(@raw_query)
+ #
+ when .subscriptions?, .regular?
+ if params["sp"]?
+ # Parse the `sp` URL parameter (youtube compatibility)
+ @filters = Filters.from_yt_params(params)
+ @query = @raw_query || ""
+ else
+ # Parse invidious URL parameters (sort, date, etc...)
+ @filters = Filters.from_iv_params(params)
+ @channel = params["channel"]? || ""
+
+ if @filters.default? && @raw_query.index(/\w:\w/)
+ # Parse legacy filters from query
+ @filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query)
+ else
+ @query = @raw_query || ""
+ end
+
+ if !@channel.empty?
+ # Switch to channel search mode (filters will be ignored)
+ @type = Type::Channel
+ elsif subs
+ # Switch to subscriptions search mode
+ @type = Type::Subscriptions
+ end
+ end
+ end
+ end
+
+ # Run the search query using the corresponding search processor.
+ # Returns either the results or an empty array of `SearchItem`.
+ def process(user : Invidious::User? = nil) : Array(SearchItem) | Array(ChannelVideo)
+ items = [] of SearchItem
+
+ # Don't bother going further if search query is empty
+ return items if self.empty_raw_query?
+
+ case @type
+ when .regular?, .playlist?
+ items = Processors.regular(self)
+ #
+ when .channel?
+ items = Processors.channel(self)
+ #
+ when .subscriptions?
+ if user
+ items = Processors.subscriptions(self, user.as(Invidious::User))
+ end
+ end
+
+ return items
+ end
+
+ # Return the HTTP::Params corresponding to this Query (invidious format)
+ def to_http_params : HTTP::Params
+ params = @filters.to_iv_params
+
+ params["q"] = @query
+ params["channel"] = @channel if !@channel.empty?
+
+ return params
+ end
+
+ # Checks if the query is a standalone URL
+ def url? : Bool
+ # If the smart features have been inhibited, don't go further.
+ return false if @inhibit_ssf
+
+ # Only supported in regular search mode
+ return false if !@type.regular?
+
+ # If filters are present, that's a regular search
+ return false if !@filters.default?
+
+ # Simple heuristics: domain name
+ return @raw_query.starts_with?(
+ /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\//
+ )
+ end
+ end
+end
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index 1f957081..d14cde5d 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -4,11 +4,12 @@ def fetch_trending(trending_type, region, locale)
plid = nil
- if trending_type == "Music"
+ case trending_type.try &.downcase
+ when "music"
params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D"
- elsif trending_type == "Gaming"
+ when "gaming"
params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D"
- elsif trending_type == "Movies"
+ when "movies"
params = "4gIKGgh0cmFpbGVycw%3D%3D"
else # Default
params = ""
@@ -16,7 +17,26 @@ def fetch_trending(trending_type, region, locale)
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.browse("FEtrending", params: params, client_config: client_config)
- trending = extract_videos(initial_data)
- return {trending, plid}
+ items, _ = extract_items(initial_data)
+
+ extracted = [] of SearchItem
+
+ deduplicate = items.size > 1
+
+ items.each do |itm|
+ if itm.is_a?(Category)
+ # Ignore the smaller categories, as they generally contain a sponsored
+ # channel, which brings a lot of noise on the trending page.
+ # See: https://github.com/iv-org/invidious/issues/2989
+ next if (itm.contents.size < 24 && deduplicate)
+
+ extracted.concat itm.contents.select(SearchItem)
+ else
+ extracted << itm
+ end
+ end
+
+ # Deduplicate items before returning results
+ return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
end
diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr
index 8a0f67e5..b175c3b9 100644
--- a/src/invidious/user/captcha.cr
+++ b/src/invidious/user/captcha.cr
@@ -4,8 +4,6 @@ struct Invidious::User
module Captcha
extend self
- private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com")
-
def generate_image(key)
second = Random::Secure.rand(12)
second_angle = second * 30
@@ -60,19 +58,5 @@ struct Invidious::User
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)},
}
end
-
- def generate_text(key)
- response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body)
- response = JSON.parse(response)
-
- tokens = response["a"].as_a.map do |answer|
- generate_response(answer.as_s, {":login"}, key, use_nonce: true)
- end
-
- return {
- question: response["q"].as_s,
- tokens: tokens,
- }
- end
end
end
diff --git a/src/invidious/user/cookies.cr b/src/invidious/user/cookies.cr
index 99df1b07..654efc15 100644
--- a/src/invidious/user/cookies.cr
+++ b/src/invidious/user/cookies.cr
@@ -17,7 +17,8 @@ struct Invidious::User
value: sid,
expires: Time.utc + 2.years,
secure: SECURE,
- http_only: true
+ http_only: true,
+ samesite: HTTP::Cookie::SameSite::Lax
)
end
@@ -30,7 +31,8 @@ struct Invidious::User
value: URI.encode_www_form(preferences.to_json),
expires: Time.utc + 2.years,
secure: SECURE,
- http_only: true
+ http_only: false,
+ samesite: HTTP::Cookie::SameSite::Lax
)
end
end
diff --git a/src/invidious/user/exports.cr b/src/invidious/user/exports.cr
new file mode 100644
index 00000000..b52503c9
--- /dev/null
+++ b/src/invidious/user/exports.cr
@@ -0,0 +1,35 @@
+struct Invidious::User
+ module Export
+ extend self
+
+ def to_invidious(user : User)
+ playlists = Invidious::Database::Playlists.select_like_iv(user.email)
+
+ return JSON.build do |json|
+ json.object do
+ json.field "subscriptions", user.subscriptions
+ json.field "watch_history", user.watched
+ json.field "preferences", user.preferences
+ json.field "playlists" do
+ json.array do
+ playlists.each do |playlist|
+ json.object do
+ json.field "title", playlist.title
+ json.field "description", html_to_content(playlist.description_html)
+ json.field "privacy", playlist.privacy.to_s
+ json.field "videos" do
+ json.array do
+ Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: CONFIG.playlist_length_limit).each do |video_id|
+ json.string video_id
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end # module
+end
diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr
index f8b9e4e4..007eb666 100644
--- a/src/invidious/user/imports.cr
+++ b/src/invidious/user/imports.cr
@@ -6,7 +6,7 @@ struct Invidious::User
# Parse a youtube CSV subscription file
def parse_subscription_export_csv(csv_content : String)
- rows = CSV.new(csv_content, headers: true)
+ rows = CSV.new(csv_content.strip('\n'), headers: true)
subscriptions = Array(String).new
# Counter to limit the amount of imports.
@@ -30,6 +30,60 @@ struct Invidious::User
return subscriptions
end
+ def parse_playlist_export_csv(user : User, raw_input : String)
+ # Split the input into head and body content
+ raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true)
+
+ # Create the playlist from the head content
+ csv_head = CSV.new(raw_head.strip('\n'), headers: true)
+ csv_head.next
+ title = csv_head[4]
+ description = csv_head[5]
+ visibility = csv_head[6]
+
+ if visibility.compare("Public", case_insensitive: true) == 0
+ privacy = PlaylistPrivacy::Public
+ else
+ privacy = PlaylistPrivacy::Private
+ end
+
+ playlist = create_playlist(title, privacy, user)
+ Invidious::Database::Playlists.update_description(playlist.id, description)
+
+ # Add each video to the playlist from the body content
+ csv_body = CSV.new(raw_body.strip('\n'), headers: true)
+ csv_body.each do |row|
+ video_id = row[0]
+ if playlist
+ next if !video_id
+ next if video_id == "Video Id"
+
+ begin
+ video = get_video(video_id)
+ rescue ex
+ next
+ end
+
+ playlist_video = PlaylistVideo.new({
+ title: video.title,
+ id: video.id,
+ author: video.author,
+ ucid: video.ucid,
+ length_seconds: video.length_seconds,
+ published: video.published,
+ plid: playlist.id,
+ live_now: video.live_now,
+ index: Random::Secure.rand(0_i64..Int64::MAX),
+ })
+
+ Invidious::Database::PlaylistVideos.insert(playlist_video)
+ Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
+ end
+ end
+
+ return playlist
+ end
+
# -------------------
# Invidious
# -------------------
@@ -48,7 +102,7 @@ struct Invidious::User
if data["watch_history"]?
user.watched += data["watch_history"].as_a.map(&.as_s)
- user.watched.uniq!
+ user.watched.reverse!.uniq!.reverse!
Invidious::Database::Users.update_watch_history(user)
end
@@ -61,7 +115,7 @@ struct Invidious::User
playlists.each do |item|
title = item["title"]?.try &.as_s?.try &.delete("<>")
description = item["description"]?.try &.as_s?.try &.delete("\r")
- privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
+ privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state }
next if !title
next if !description
@@ -70,14 +124,16 @@ struct Invidious::User
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
- videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
- raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
+ item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
+ if idx > CONFIG.playlist_length_limit
+ raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
+ end
video_id = video_id.try &.as_s?
next if !video_id
begin
- video = get_video(video_id)
+ video = get_video(video_id, false)
rescue ex
next
end
@@ -105,7 +161,7 @@ struct Invidious::User
# Youtube
# -------------------
- private def is_opml?(mimetype : String, extension : String)
+ private def opml?(mimetype : String, extension : String)
opml_mimetypes = [
"application/xml",
"text/xml",
@@ -123,10 +179,10 @@ struct Invidious::User
def from_youtube(user : User, body : String, filename : String, type : String) : Bool
extension = filename.split(".").last
- if is_opml?(type, extension)
+ if opml?(type, extension)
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
- channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
+ channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0]
end
elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body)
@@ -147,6 +203,41 @@ struct Invidious::User
return true
end
+ def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool
+ extension = filename.split(".").last
+
+ if extension == "csv" || type == "text/csv"
+ playlist = parse_playlist_export_csv(user, body)
+ if playlist
+ return true
+ else
+ return false
+ end
+ else
+ return false
+ end
+ end
+
+ def from_youtube_wh(user : User, body : String, filename : String, type : String) : Bool
+ extension = filename.split(".").last
+
+ if extension == "json" || type == "application/json"
+ data = JSON.parse(body)
+ watched = data.as_a.compact_map do |item|
+ next unless url = item["titleUrl"]?
+ next unless match = url.as_s.match(/\?v=(?[a-zA-Z0-9_-]+)$/)
+ match["video_id"]
+ end
+ watched.reverse! # YouTube have newest first
+ user.watched += watched
+ user.watched.uniq!
+ Invidious::Database::Users.update_watch_history(user)
+ return true
+ else
+ return false
+ end
+ end
+
# -------------------
# Freetube
# -------------------
@@ -157,8 +248,12 @@ struct Invidious::User
subs = matches.map(&.["channel_id"])
if subs.empty?
- data = JSON.parse(body)["subscriptions"]
- subs = data.as_a.map(&.["id"].as_s)
+ profiles = body.split('\n', remove_empty: true)
+ profiles.each do |profile|
+ if data = JSON.parse(profile)["subscriptions"]?
+ subs += data.as_a.map(&.["id"].as_s)
+ end
+ end
end
user.subscriptions += subs
@@ -195,42 +290,39 @@ struct Invidious::User
end
def from_newpipe(user : User, body : String) : Bool
- io = IO::Memory.new(body)
+ Compress::Zip::File.open(IO::Memory.new(body), true) do |file|
+ entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" }
+ return false if entry.nil?
+ entry.open do |file_io|
+ # Ensure max size of 4MB
+ io_sized = IO::Sized.new(file_io, 0x400000)
- Compress::Zip::File.open(io) do |file|
- file.entries.each do |entry|
- entry.open do |file_io|
- # Ensure max size of 4MB
- io_sized = IO::Sized.new(file_io, 0x400000)
+ begin
+ temp = File.tempfile(".db") do |tempfile|
+ begin
+ File.write(tempfile.path, io_sized.gets_to_end)
+ rescue
+ return false
+ end
- next if entry.filename != "newpipe.db"
+ DB.open("sqlite3://" + tempfile.path) do |db|
+ user.watched += db.query_all("SELECT url FROM streams", as: String)
+ .map(&.lchop("https://www.youtube.com/watch?v="))
- tempfile = File.tempfile(".db")
+ user.watched.uniq!
+ Invidious::Database::Users.update_watch_history(user)
- begin
- File.write(tempfile.path, io_sized.gets_to_end)
- rescue
- return false
+ user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
+ .map(&.lchop("https://www.youtube.com/channel/"))
+
+ user.subscriptions.uniq!
+ user.subscriptions = get_batch_channels(user.subscriptions)
+
+ Invidious::Database::Users.update_subscriptions(user)
+ end
end
-
- db = DB.open("sqlite3://" + tempfile.path)
-
- user.watched += db.query_all("SELECT url FROM streams", as: String)
- .map(&.lchop("https://www.youtube.com/watch?v="))
-
- user.watched.uniq!
- Invidious::Database::Users.update_watch_history(user)
-
- user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
- .map(&.lchop("https://www.youtube.com/channel/"))
-
- user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions)
-
- Invidious::Database::Users.update_subscriptions(user)
-
- db.close
- tempfile.delete
+ ensure
+ temp.delete if !temp.nil?
end
end
end
diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr
index b3059403..0a8525f3 100644
--- a/src/invidious/user/preferences.cr
+++ b/src/invidious/user/preferences.cr
@@ -4,6 +4,7 @@ struct Preferences
property annotations : Bool = CONFIG.default_user_preferences.annotations
property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
+ property preload : Bool = CONFIG.default_user_preferences.preload
property autoplay : Bool = CONFIG.default_user_preferences.autoplay
property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index b763596b..65566d20 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -3,75 +3,6 @@ require "crypto/bcrypt/password"
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
-def get_user(sid, headers, refresh = true)
- if email = Invidious::Database::SessionIDs.select_email(sid)
- user = Invidious::Database::Users.select!(email: email)
-
- if refresh && Time.utc - user.updated > 1.minute
- user, sid = fetch_user(sid, headers)
-
- Invidious::Database::Users.insert(user, update_on_conflict: true)
- Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
-
- begin
- view_name = "subscriptions_#{sha256(user.email)}"
- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
- rescue ex
- end
- end
- else
- user, sid = fetch_user(sid, headers)
-
- Invidious::Database::Users.insert(user, update_on_conflict: true)
- Invidious::Database::SessionIDs.insert(sid, user.email, handle_conflicts: true)
-
- begin
- view_name = "subscriptions_#{sha256(user.email)}"
- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
- rescue ex
- end
- end
-
- return user, sid
-end
-
-def fetch_user(sid, headers)
- feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
- feed = XML.parse_html(feed.body)
-
- channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
- if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
- nil
- else
- channel["href"].lstrip("/channel/")
- end
- end
-
- channels = get_batch_channels(channels)
-
- email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
- if email
- email = email.content.strip
- else
- email = ""
- end
-
- token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
-
- user = Invidious::User.new({
- updated: Time.utc,
- notifications: [] of String,
- subscriptions: channels,
- email: email,
- preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
- password: nil,
- token: token,
- watched: [] of String,
- feed_needs_update: true,
- })
- return user, sid
-end
-
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
@@ -91,38 +22,6 @@ def create_user(sid, email, password)
return user, sid
end
-def subscribe_ajax(channel_id, action, env_headers)
- headers = HTTP::Headers.new
- headers["Cookie"] = env_headers["Cookie"]
-
- html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
-
- cookies = HTTP::Cookies.from_client_headers(headers)
- html.cookies.each do |cookie|
- if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
- if cookies[cookie.name]?
- cookies[cookie.name] = cookie
- else
- cookies << cookie
- end
- end
- end
- headers = cookies.add_request_headers(headers)
-
- if match = html.body.match(/'XSRF_TOKEN': "(?[^"]+)"/)
- session_token = match["session_token"]
-
- headers["content-type"] = "application/x-www-form-urlencoded"
-
- post_req = {
- session_token: session_token,
- }
- post_url = "/subscription_ajax?#{action}=1&c=#{channel_id}"
-
- YT_POOL.client &.post(post_url, headers, form: post_req)
- end
-end
-
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 81fce5b8..348a0a66 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -1,280 +1,22 @@
-CAPTION_LANGUAGES = {
- "",
- "English",
- "English (auto-generated)",
- "English (United Kingdom)",
- "English (United States)",
- "Afrikaans",
- "Albanian",
- "Amharic",
- "Arabic",
- "Armenian",
- "Azerbaijani",
- "Bangla",
- "Basque",
- "Belarusian",
- "Bosnian",
- "Bulgarian",
- "Burmese",
- "Cantonese (Hong Kong)",
- "Catalan",
- "Cebuano",
- "Chinese",
- "Chinese (China)",
- "Chinese (Hong Kong)",
- "Chinese (Simplified)",
- "Chinese (Taiwan)",
- "Chinese (Traditional)",
- "Corsican",
- "Croatian",
- "Czech",
- "Danish",
- "Dutch",
- "Dutch (auto-generated)",
- "Esperanto",
- "Estonian",
- "Filipino",
- "Finnish",
- "French",
- "French (auto-generated)",
- "Galician",
- "Georgian",
- "German",
- "German (auto-generated)",
- "Greek",
- "Gujarati",
- "Haitian Creole",
- "Hausa",
- "Hawaiian",
- "Hebrew",
- "Hindi",
- "Hmong",
- "Hungarian",
- "Icelandic",
- "Igbo",
- "Indonesian",
- "Indonesian (auto-generated)",
- "Interlingue",
- "Irish",
- "Italian",
- "Italian (auto-generated)",
- "Japanese",
- "Japanese (auto-generated)",
- "Javanese",
- "Kannada",
- "Kazakh",
- "Khmer",
- "Korean",
- "Korean (auto-generated)",
- "Kurdish",
- "Kyrgyz",
- "Lao",
- "Latin",
- "Latvian",
- "Lithuanian",
- "Luxembourgish",
- "Macedonian",
- "Malagasy",
- "Malay",
- "Malayalam",
- "Maltese",
- "Maori",
- "Marathi",
- "Mongolian",
- "Nepali",
- "Norwegian Bokmål",
- "Nyanja",
- "Pashto",
- "Persian",
- "Polish",
- "Portuguese",
- "Portuguese (auto-generated)",
- "Portuguese (Brazil)",
- "Punjabi",
- "Romanian",
- "Russian",
- "Russian (auto-generated)",
- "Samoan",
- "Scottish Gaelic",
- "Serbian",
- "Shona",
- "Sindhi",
- "Sinhala",
- "Slovak",
- "Slovenian",
- "Somali",
- "Southern Sotho",
- "Spanish",
- "Spanish (auto-generated)",
- "Spanish (Latin America)",
- "Spanish (Mexico)",
- "Spanish (Spain)",
- "Sundanese",
- "Swahili",
- "Swedish",
- "Tajik",
- "Tamil",
- "Telugu",
- "Thai",
- "Turkish",
- "Turkish (auto-generated)",
- "Ukrainian",
- "Urdu",
- "Uzbek",
- "Vietnamese",
- "Vietnamese (auto-generated)",
- "Welsh",
- "Western Frisian",
- "Xhosa",
- "Yiddish",
- "Yoruba",
- "Zulu",
-}
-
-REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"}
-
-# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
-VIDEO_FORMATS = {
- "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
- "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
- "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
- "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
- "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
- "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
-
- "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
- "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
-
- # 3D videos
- "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
- "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
- "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
- "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
-
- # Apple HTTP Live Streaming
- "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
- "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
- "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
- "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
- "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
-
- # DASH mp4 video
- "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
- "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
- "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
- "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
- "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
- "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
- "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
- "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
- "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
- "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
- "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
- "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
-
- # Dash mp4 audio
- "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
- "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
- "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
- "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
- "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
- "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
- "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
-
- # Dash webm
- "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
- "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
- "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
- "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
- "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
- "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
- "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
- "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
- # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
- "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
- "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
- "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
- "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
-
- # Dash webm audio
- "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
- "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
-
- # Dash webm audio with opus inside
- "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
- "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
- "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
-
- # av01 video only formats sometimes served with "unknown" codecs
- "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
- "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
- "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
- "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
-}
-
-struct VideoPreferences
- include JSON::Serializable
-
- property annotations : Bool
- property autoplay : Bool
- property comments : Array(String)
- property continue : Bool
- property continue_autoplay : Bool
- property controls : Bool
- property listen : Bool
- property local : Bool
- property preferred_captions : Array(String)
- property player_style : String
- property quality : String
- property quality_dash : String
- property raw : Bool
- property region : String?
- property related_videos : Bool
- property speed : Float32 | Float64
- property video_end : Float64 | Int32
- property video_loop : Bool
- property extend_desc : Bool
- property video_start : Float64 | Int32
- property volume : Int32
- property vr_mode : Bool
- property save_player_pos : Bool
+enum VideoType
+ Video
+ Livestream
+ Scheduled
end
struct Video
include DB::Serializable
+ # Version of the JSON structure
+ # It prevents us from loading an incompatible version from cache
+ # (either newer or older, if instances with different versions run
+ # concurrently, e.g during a version upgrade rollout).
+ #
+ # NOTE: don't forget to bump this number if any change is made to
+ # the `params` structure in videos/parser.cr!!!
+ #
+ SCHEMA_VERSION = 3
+
property id : String
@[DB::Field(converter: Video::JSONConverter)]
@@ -282,13 +24,7 @@ struct Video
property updated : Time
@[DB::Field(ignore: true)]
- property captions : Array(Caption)?
-
- @[DB::Field(ignore: true)]
- property adaptive_fmts : Array(Hash(String, JSON::Any))?
-
- @[DB::Field(ignore: true)]
- property fmt_stream : Array(Hash(String, JSON::Any))?
+ @captions = [] of Invidious::Videos::Captions::Metadata
@[DB::Field(ignore: true)]
property description : String?
@@ -299,274 +35,49 @@ struct Video
end
end
+ # Methods for API v1 JSON
+
def to_json(locale : String?, json : JSON::Builder)
- json.object do
- json.field "type", "video"
-
- json.field "title", self.title
- json.field "videoId", self.id
-
- json.field "error", info["reason"] if info["reason"]?
-
- json.field "videoThumbnails" do
- generate_thumbnails(json, self.id)
- end
- json.field "storyboards" do
- generate_storyboards(json, self.id, self.storyboards)
- end
-
- json.field "description", self.description
- json.field "descriptionHtml", self.description_html
- json.field "published", self.published.to_unix
- json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
- json.field "keywords", self.keywords
-
- json.field "viewCount", self.views
- json.field "likeCount", self.likes
- json.field "dislikeCount", self.dislikes
-
- json.field "paid", self.paid
- json.field "premium", self.premium
- json.field "isFamilyFriendly", self.is_family_friendly
- json.field "allowedRegions", self.allowed_regions
- json.field "genre", self.genre
- json.field "genreUrl", self.genre_url
-
- json.field "author", self.author
- json.field "authorId", self.ucid
- json.field "authorUrl", "/channel/#{self.ucid}"
-
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
-
- json.field "subCountText", self.sub_count_text
-
- json.field "lengthSeconds", self.length_seconds
- json.field "allowRatings", self.allow_ratings
- json.field "rating", self.average_rating
- json.field "isListed", self.is_listed
- json.field "liveNow", self.live_now
- json.field "isUpcoming", self.is_upcoming
-
- if self.premiere_timestamp
- json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
- end
-
- if hlsvp = self.hls_manifest_url
- hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
- json.field "hlsUrl", hlsvp
- end
-
- json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
-
- json.field "adaptiveFormats" do
- json.array do
- self.adaptive_fmts.each do |fmt|
- json.object do
- json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}"
- json.field "bitrate", fmt["bitrate"].as_i.to_s
- json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}"
- json.field "url", fmt["url"]
- json.field "itag", fmt["itag"].as_i.to_s
- json.field "type", fmt["mimeType"]
- json.field "clen", fmt["contentLength"]
- json.field "lmt", fmt["lastModified"]
- json.field "projectionType", fmt["projectionType"]
-
- fmt_info = itag_to_metadata?(fmt["itag"])
- if fmt_info
- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
- json.field "fps", fps
- json.field "container", fmt_info["ext"]
- json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
- if fmt_info["height"]?
- json.field "resolution", "#{fmt_info["height"]}p"
-
- quality_label = "#{fmt_info["height"]}p"
- if fps > 30
- quality_label += "60"
- end
- json.field "qualityLabel", quality_label
-
- if fmt_info["width"]?
- json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
- end
- end
- end
- end
- end
- end
- end
-
- json.field "formatStreams" do
- json.array do
- self.fmt_stream.each do |fmt|
- json.object do
- json.field "url", fmt["url"]
- json.field "itag", fmt["itag"].as_i.to_s
- json.field "type", fmt["mimeType"]
- json.field "quality", fmt["quality"]
-
- fmt_info = itag_to_metadata?(fmt["itag"])
- if fmt_info
- fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
- json.field "fps", fps
- json.field "container", fmt_info["ext"]
- json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
- if fmt_info["height"]?
- json.field "resolution", "#{fmt_info["height"]}p"
-
- quality_label = "#{fmt_info["height"]}p"
- if fps > 30
- quality_label += "60"
- end
- json.field "qualityLabel", quality_label
-
- if fmt_info["width"]?
- json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
- end
- end
- end
- end
- end
- end
- end
-
- json.field "captions" do
- json.array do
- self.captions.each do |caption|
- json.object do
- json.field "label", caption.name
- json.field "language_code", caption.language_code
- json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}"
- end
- end
- end
- end
-
- json.field "recommendedVideos" do
- json.array do
- self.related_videos.each do |rv|
- if rv["id"]?
- json.object do
- json.field "videoId", rv["id"]
- json.field "title", rv["title"]
- json.field "videoThumbnails" do
- generate_thumbnails(json, rv["id"])
- end
-
- json.field "author", rv["author"]
- json.field "authorUrl", "/channel/#{rv["ucid"]?}"
- json.field "authorId", rv["ucid"]?
- if rv["author_thumbnail"]?
- json.field "authorThumbnails" do
- json.array do
- qualities = {32, 48, 76, 100, 176, 512}
-
- qualities.each do |quality|
- json.object do
- json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
- json.field "width", quality
- json.field "height", quality
- end
- end
- end
- end
- end
-
- json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
- json.field "viewCountText", rv["short_view_count"]?
- json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
- end
- end
- end
- end
- end
- end
+ Invidious::JSONify::APIv1.video(self, json, locale: locale)
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
- JSON.build { |json| to_json(locale, json) }
+ JSON.build do |json|
+ Invidious::JSONify::APIv1.video(self, json, locale: locale)
+ end
end
def to_json(json : JSON::Builder | Nil = nil)
to_json(nil, json)
end
- def title
- info["videoDetails"]["title"]?.try &.as_s || ""
+ # Misc methods
+
+ def video_type : VideoType
+ video_type = info["videoType"]?.try &.as_s || "video"
+ return VideoType.parse?(video_type) || VideoType::Video
end
- def ucid
- info["videoDetails"]["channelId"]?.try &.as_s || ""
- end
-
- def author
- info["videoDetails"]["author"]?.try &.as_s || ""
- end
-
- def length_seconds : Int32
- info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i ||
- info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
- end
-
- def views : Int64
- info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
- end
-
- def likes : Int64
- info["likes"]?.try &.as_i64 || 0_i64
- end
-
- def dislikes : Int64
- info["dislikes"]?.try &.as_i64 || 0_i64
- end
-
- def average_rating : Float64
- # (likes / (likes + dislikes) * 4 + 1)
- info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0
+ def schema_version : Int
+ return info["version"]?.try &.as_i || 1
end
def published : Time
- info
- .dig?("microformat", "playerMicroformatRenderer", "publishDate")
+ return info["published"]?
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
end
def published=(other : Time)
- info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
- end
-
- def allow_ratings
- r = info["videoDetails"]["allowRatings"]?.try &.as_bool
- r.nil? ? false : r
+ info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
end
def live_now
- info["microformat"]?.try &.["playerMicroformatRenderer"]?
- .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false
+ return (self.video_type == VideoType::Livestream)
end
- def is_listed
- info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
- end
-
- def is_upcoming
- info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
+ def post_live_dvr
+ return info["isPostLiveDvr"].as_bool
end
def premiere_timestamp : Time?
@@ -575,68 +86,30 @@ struct Video
.try { |t| Time.parse_rfc3339(t.as_s) }
end
- def keywords
- info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
- end
-
def related_videos
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
end
- def allowed_regions
- info
- .dig("microformat", "playerMicroformatRenderer", "availableCountries")
- .try &.as_a.map &.as_s || [] of String
- end
+ # Methods for parsing streaming data
- def author_thumbnail : String
- info["authorThumbnail"]?.try &.as_s || ""
- end
-
- def sub_count_text : String
- info["subCountText"]?.try &.as_s || "-"
- end
-
- def fmt_stream
- return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
-
- fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
- fmt_stream.each do |fmt|
- if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
- s.each do |k, v|
- fmt[k] = JSON::Any.new(v)
- end
- fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
- end
-
- fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
- fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
+ def fmt_stream : Array(Hash(String, JSON::Any))
+ if formats = info.dig?("streamingData", "formats")
+ return formats
+ .as_a.map(&.as_h)
+ .sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ else
+ return [] of Hash(String, JSON::Any)
end
- fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
- @fmt_stream = fmt_stream
- return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
- def adaptive_fmts
- return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
- fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
- fmt_stream.each do |fmt|
- if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
- s.each do |k, v|
- fmt[k] = JSON::Any.new(v)
- end
- fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
- end
-
- fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
- fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
+ def adaptive_fmts : Array(Hash(String, JSON::Any))
+ if formats = info.dig?("streamingData", "adaptiveFormats")
+ return formats
+ .as_a.map(&.as_h)
+ .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 }
+ else
+ return [] of Hash(String, JSON::Any)
end
- # See https://github.com/TeamNewPipe/NewPipe/issues/2415
- # Some streams are segmented by URL `sq/` rather than index, for now we just filter them out
- fmt_stream.reject! { |f| !f["indexRange"]? }
- fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
- @adaptive_fmts = fmt_stream
- return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
@@ -647,429 +120,178 @@ struct Video
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
+ # Misc. methods
+
def storyboards
- storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
- .try &.as_s.split("|")
-
- if !storyboards
- if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
- return [{
- url: storyboard.split("#")[0],
- width: 106,
- height: 60,
- count: -1,
- interval: 5000,
- storyboard_width: 3,
- storyboard_height: 3,
- storyboard_count: -1,
- }]
- end
- end
-
- items = [] of NamedTuple(
- url: String,
- width: Int32,
- height: Int32,
- count: Int32,
- interval: Int32,
- storyboard_width: Int32,
- storyboard_height: Int32,
- storyboard_count: Int32)
-
- return items if !storyboards
-
- url = URI.parse(storyboards.shift)
- params = HTTP::Params.parse(url.query || "")
-
- storyboards.each_with_index do |sb, i|
- width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
- params["sigh"] = sigh
- url.query = params.to_s
-
- width = width.to_i
- height = height.to_i
- count = count.to_i
- interval = interval.to_i
- storyboard_width = storyboard_width.to_i
- storyboard_height = storyboard_height.to_i
- storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
-
- items << {
- url: url.to_s.sub("$L", i).sub("$N", "M$M"),
- width: width,
- height: height,
- count: count,
- interval: interval,
- storyboard_width: storyboard_width,
- storyboard_height: storyboard_height,
- storyboard_count: storyboard_count,
- }
- end
-
- items
+ container = info.dig?("storyboards") || JSON::Any.new("{}")
+ return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds)
end
def paid
- reason = info.dig?("playabilityStatus", "reason").try &.as_s || ""
- return reason.includes? "requires payment"
+ return (self.reason || "").includes? "requires payment"
end
def premium
keywords.includes? "YouTube Red"
end
- def captions : Array(Caption)
- return @captions.as(Array(Caption)) if @captions
- captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
- name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
- language_code = caption["languageCode"].to_s
- base_url = caption["baseUrl"].to_s
-
- caption = Caption.new(name.to_s, language_code, base_url)
- caption.name = caption.name.split(" - ")[0]
- caption
+ def captions : Array(Invidious::Videos::Captions::Metadata)
+ if @captions.empty? && @info.has_key?("captions")
+ @captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"])
end
- captions ||= [] of Caption
- @captions = captions
- return @captions.as(Array(Caption))
- end
- def description
- description = info
- .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText")
- .try &.as_s || ""
- end
-
- # TODO
- def description=(value : String)
- @description = value
- end
-
- def description_html
- info["descriptionHtml"]?.try &.as_s || ""
- end
-
- def description_html=(value : String)
- info["descriptionHtml"] = JSON::Any.new(value)
- end
-
- def short_description
- info["shortDescription"]?.try &.as_s? || ""
+ return @captions
end
def hls_manifest_url : String?
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
end
- def dash_manifest_url
- info.dig?("streamingData", "dashManifestUrl").try &.as_s
- end
+ def dash_manifest_url : String?
+ raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s
+ return nil if raw_dash_url.nil?
- def genre : String
- info["genre"]?.try &.as_s || ""
+ # Use manifest v5 parameter to reduce file size
+ # See https://github.com/iv-org/invidious/issues/4186
+ dash_url = URI.parse(raw_dash_url)
+ dash_query = dash_url.query || ""
+
+ if dash_query.empty?
+ dash_url.path = "#{dash_url.path}/mpd_version/5"
+ else
+ dash_url.query = "#{dash_query}&mpd_version=5"
+ end
+
+ return dash_url.to_s
end
def genre_url : String?
- info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
+ info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
end
- def license : String?
- info["license"]?.try &.as_s
- end
-
- def is_family_friendly : Bool
- info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false
- end
-
- def is_vr : Bool?
- projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
- return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
+ def vr? : Bool?
+ return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
end
def projection_type : String?
return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
end
- def wilson_score : Float64
- ci_lower_bound(likes, likes + dislikes).round(4)
- end
-
- def engagement : Float64
- (((likes + dislikes) / views) * 100).round(4)
- end
-
def reason : String?
info["reason"]?.try &.as_s
end
-end
-struct Caption
- property name
- property language_code
- property base_url
-
- getter name : String
- getter language_code : String
- getter base_url : String
-
- setter name
-
- def initialize(@name, @language_code, @base_url)
- end
-end
-
-class VideoRedirect < Exception
- property video_id : String
-
- def initialize(@video_id)
- end
-end
-
-# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
-# The former is preferred as it has more videos in it. The second has
-# the same 11 first entries as the compact rendered.
-#
-# TODO: "compactRadioRenderer" (Mix) and
-def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
- return nil if !related["videoId"]?
-
- # The compact renderer has video length in seconds, where the end
- # screen rendered has a full text version ("42:40")
- length = related["lengthInSeconds"]?.try &.as_i.to_s
- length ||= related.dig?("lengthText", "simpleText").try do |box|
- decode_length_seconds(box.as_s).to_s
+ def music : Array(VideoMusic)
+ info["music"].as_a.map { |music_json|
+ VideoMusic.new(
+ music_json["song"].as_s,
+ music_json["album"].as_s,
+ music_json["artist"].as_s,
+ music_json["license"].as_s
+ )
+ }
end
- # Both have "short", so the "long" option shouldn't be required
- channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
- .try &.dig?("runs", 0)
+ # Macros defining getters/setters for various types of data
- author = channel_info.try &.dig?("text")
- ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
-
- # "4,088,033 views", only available on compact renderer
- # and when video is not a livestream
- view_count = related.dig?("viewCountText", "simpleText")
- .try &.as_s.gsub(/\D/, "")
-
- short_view_count = related.try do |r|
- HelperExtractors.get_short_view_count(r).to_s
- end
-
- LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
-
- # TODO: when refactoring video types, make a struct for related videos
- # or reuse an existing type, if that fits.
- return {
- "id" => related["videoId"],
- "title" => related["title"]["simpleText"],
- "author" => author || JSON::Any.new(""),
- "ucid" => JSON::Any.new(ucid || ""),
- "length_seconds" => JSON::Any.new(length || "0"),
- "view_count" => JSON::Any.new(view_count || "0"),
- "short_view_count" => JSON::Any.new(short_view_count || "0"),
- }
-end
-
-def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil)
- params = {} of String => JSON::Any
-
- client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
- if context_screen == "embed"
- client_config.client_type = YoutubeAPI::ClientType::WebScreenEmbed
- end
-
- player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
-
- if player_response.dig?("playabilityStatus", "status").try &.as_s != "OK"
- subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
- reason = subreason.try &.[]?("simpleText").try &.as_s
- reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
- reason ||= player_response.dig("playabilityStatus", "reason").as_s
- params["reason"] = JSON::Any.new(reason)
- return params
- end
-
- params["shortDescription"] = player_response.dig?("videoDetails", "shortDescription") || JSON::Any.new(nil)
-
- # Don't fetch the next endpoint if the video is unavailable.
- if !params["reason"]?
- next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
- player_response = player_response.merge(next_response)
- end
-
- # Fetch the video streams using an Android client in order to get the decrypted URLs and
- # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs:
- # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
- if !params["reason"]?
- if context_screen == "embed"
- client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
- else
- client_config.client_type = YoutubeAPI::ClientType::Android
+ private macro getset_string(name)
+ # Return {{name.stringify}} from `info`
+ def {{name.id.underscore}} : String
+ return info[{{name.stringify}}]?.try &.as_s || ""
end
- android_player = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
- # Sometime, the video is available from the web client, but not on Android, so check
- # that here, and fallback to the streaming data from the web client if needed.
- # See: https://github.com/iv-org/invidious/issues/2549
- if android_player["playabilityStatus"]["status"] == "OK"
- params["streamingData"] = android_player["streamingData"]? || JSON::Any.new("")
- else
- params["streamingData"] = player_response["streamingData"]? || JSON::Any.new("")
+ # Update {{name.stringify}} into `info`
+ def {{name.id.underscore}}=(value : String)
+ info[{{name.stringify}}] = JSON::Any.new(value)
end
+
+ {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- {"captions", "microformat", "playabilityStatus", "storyboards", "videoDetails"}.each do |f|
- params[f] = player_response[f] if player_response[f]?
- end
-
- # Top level elements
-
- main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
-
- raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
-
- primary_results = main_results.dig?("results", "results", "contents")
-
- raise BrokenTubeException.new("results") if !primary_results
-
- video_primary_renderer = primary_results
- .as_a.find(&.["videoPrimaryInfoRenderer"]?)
- .try &.["videoPrimaryInfoRenderer"]
-
- video_secondary_renderer = primary_results
- .as_a.find(&.["videoSecondaryInfoRenderer"]?)
- .try &.["videoSecondaryInfoRenderer"]
-
- raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
- raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
-
- # Related videos
-
- LOGGER.debug("extract_video_info: parsing related videos...")
-
- related = [] of JSON::Any
-
- # Parse "compactVideoRenderer" items (under secondary results)
- secondary_results = main_results
- .dig?("secondaryResults", "secondaryResults", "results")
- secondary_results.try &.as_a.each do |element|
- if item = element["compactVideoRenderer"]?
- related_video = parse_related_video(item)
- related << JSON::Any.new(related_video) if related_video
+ private macro getset_string_array(name)
+ # Return {{name.stringify}} from `info`
+ def {{name.id.underscore}} : Array(String)
+ return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String
end
+
+ # Update {{name.stringify}} into `info`
+ def {{name.id.underscore}}=(value : Array(String))
+ info[{{name.stringify}}] = JSON::Any.new(value)
+ end
+
+ {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- # If nothing was found previously, fall back to end screen renderer
- if related.empty?
- # Container for "endScreenVideoRenderer" items
- player_overlays = player_response.dig?(
- "playerOverlays", "playerOverlayRenderer",
- "endScreen", "watchNextEndScreenRenderer", "results"
- )
-
- player_overlays.try &.as_a.each do |element|
- if item = element["endScreenVideoRenderer"]?
- related_video = parse_related_video(item)
- related << JSON::Any.new(related_video) if related_video
+ {% for op, type in {i32: Int32, i64: Int64} %}
+ private macro getset_{{op}}(name)
+ def \{{name.id.underscore}} : {{type}}
+ return info[\{{name.stringify}}]?.try &.as_i64.to_{{op}} || 0_{{op}}
end
+
+ def \{{name.id.underscore}}=(value : Int)
+ info[\{{name.stringify}}] = JSON::Any.new(value.to_i64)
+ end
+
+ \{% if flag?(:debug_macros) %} \{{debug}} \{% end %}
end
+ {% end %}
+
+ private macro getset_bool(name)
+ # Return {{name.stringify}} from `info`
+ def {{name.id.underscore}} : Bool
+ return info[{{name.stringify}}]?.try &.as_bool || false
+ end
+
+ # Update {{name.stringify}} into `info`
+ def {{name.id.underscore}}=(value : Bool)
+ info[{{name.stringify}}] = JSON::Any.new(value)
+ end
+
+ {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- params["relatedVideos"] = JSON::Any.new(related)
-
- # Likes/dislikes
-
- toplevel_buttons = video_primary_renderer
- .try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
-
- if toplevel_buttons
- likes_button = toplevel_buttons.as_a
- .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "LIKE")
- .try &.["toggleButtonRenderer"]
-
- if likes_button
- likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?)
- .try &.dig?("accessibility", "accessibilityData", "label")
- likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
-
- LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
- LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
+ # Macro to generate ? and = accessor methods for attributes in `info`
+ private macro predicate_bool(method_name, name)
+ # Return {{name.stringify}} from `info`
+ def {{method_name.id.underscore}}? : Bool
+ return info[{{name.stringify}}]?.try &.as_bool || false
end
- dislikes_button = toplevel_buttons.as_a
- .find(&.dig("toggleButtonRenderer", "defaultIcon", "iconType").as_s.== "DISLIKE")
- .try &.["toggleButtonRenderer"]
-
- if dislikes_button
- dislikes_txt = (dislikes_button["defaultText"]? || dislikes_button["toggledText"]?)
- .try &.dig?("accessibility", "accessibilityData", "label")
- dislikes = dislikes_txt.as_s.gsub(/\D/, "").to_i64? if dislikes_txt
-
- LOGGER.trace("extract_video_info: Found \"dislikes\" button. Button text is \"#{dislikes_txt}\"")
- LOGGER.debug("extract_video_info: Dislikes count is #{dislikes}") if dislikes
+ # Update {{name.stringify}} into `info`
+ def {{method_name.id.underscore}}=(value : Bool)
+ info[{{name.stringify}}] = JSON::Any.new(value)
end
+
+ {% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- if likes && likes != 0_i64 && (!dislikes || dislikes == 0_i64)
- if rating = player_response.dig?("videoDetails", "averageRating").try { |x| x.as_i64? || x.as_f? }
- dislikes = (likes * ((5 - rating)/(rating - 1))).round.to_i64
- LOGGER.debug("extract_video_info: Dislikes count (using fallback method) is #{dislikes}")
- end
- end
+ # Method definitions, using the macros above
- params["likes"] = JSON::Any.new(likes || 0_i64)
- params["dislikes"] = JSON::Any.new(dislikes || 0_i64)
+ getset_string author
+ getset_string authorThumbnail
+ getset_string description
+ getset_string descriptionHtml
+ getset_string genre
+ getset_string genreUcid
+ getset_string license
+ getset_string shortDescription
+ getset_string subCountText
+ getset_string title
+ getset_string ucid
- # Description
+ getset_string_array allowedRegions
+ getset_string_array keywords
- description_html = video_secondary_renderer.try &.dig?("description", "runs")
- .try &.as_a.try { |t| content_to_comment_html(t) }
+ getset_i32 lengthSeconds
+ getset_i64 likes
+ getset_i64 views
- params["descriptionHtml"] = JSON::Any.new(description_html || "")
-
- # Video metadata
-
- metadata = video_secondary_renderer
- .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
- .try &.as_a
-
- params["genre"] = params["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["category"]? || JSON::Any.new("")
- params["genreUrl"] = JSON::Any.new(nil)
-
- metadata.try &.each do |row|
- title = row["metadataRowRenderer"]?.try &.["title"]?.try &.["simpleText"]?.try &.as_s
- contents = row.dig?("metadataRowRenderer", "contents", 0)
-
- if title.try &.== "Category"
- contents = contents.try &.dig?("runs", 0)
-
- params["genre"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "")
- params["genreUcid"] = JSON::Any.new(contents.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]?
- .try &.["browseId"]?.try &.as_s || "")
- elsif title.try &.== "License"
- contents = contents.try &.["runs"]?
- .try &.as_a[0]?
-
- params["license"] = JSON::Any.new(contents.try &.["text"]?.try &.as_s || "")
- elsif title.try &.== "Licensed to YouTube by"
- params["license"] = JSON::Any.new(contents.try &.["simpleText"]?.try &.as_s || "")
- end
- end
-
- # Author infos
-
- author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
- author_thumbnail = author_info.try &.dig?("thumbnail", "thumbnails", 0, "url")
-
- params["authorThumbnail"] = JSON::Any.new(author_thumbnail.try &.as_s || "")
-
- params["subCountText"] = JSON::Any.new(author_info.try &.["subscriberCountText"]?
- .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }.try &.as_s.split(" ", 2)[0] || "-")
-
- # Return data
-
- return params
+ # TODO: Make predicate_bool the default as to adhere to Crystal conventions
+ getset_bool allowRatings
+ getset_bool authorVerified
+ getset_bool isFamilyFriendly
+ getset_bool isListed
+ predicate_bool upcoming, isUpcoming
end
def get_video(id, refresh = true, region = nil, force_refresh = false)
@@ -1079,7 +301,8 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
if (refresh &&
(Time.utc - video.updated > 10.minutes) ||
(video.premiere_timestamp.try &.< Time.utc)) ||
- force_refresh
+ force_refresh ||
+ video.schema_version != Video::SCHEMA_VERSION # cache control
begin
video = fetch_video(id, region)
Invidious::Database::Videos.update(video)
@@ -1094,34 +317,23 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
end
return video
+rescue DB::Error
+ # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
+ # Note: All DB errors inherit from `DB::Error`
+ return fetch_video(id, region)
end
def fetch_video(id, region)
info = extract_video_info(video_id: id)
- allowed_regions = info
- .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
- .try &.as_a.map &.as_s || [] of String
-
- # Check for region-blocks
- if info["reason"]?.try &.as_s.includes?("your country")
- bypass_regions = PROXY_LIST.keys & allowed_regions
- if !bypass_regions.empty?
- region = bypass_regions[rand(bypass_regions.size)]
- region_info = extract_video_info(video_id: id, proxy_region: region)
- region_info["region"] = JSON::Any.new(region) if region
- info = region_info if !region_info["reason"]?
- end
- end
-
- # Try to fetch video info using an embedded client
- if info["reason"]?
- embed_info = extract_video_info(video_id: id, context_screen: "embed")
- info = embed_info if !embed_info["reason"]?
- end
-
if reason = info["reason"]?
- raise InfoException.new(reason.as_s || "")
+ if reason == "Video unavailable"
+ raise NotFoundException.new(reason.as_s || "")
+ elsif !reason.as_s.starts_with? "Premieres"
+ # dont error when it's a premiere.
+ # we already parsed most of the data and display the premiere date
+ raise InfoException.new(reason.as_s || "")
+ end
end
video = Video.new({
@@ -1133,10 +345,6 @@ def fetch_video(id, region)
return video
end
-def itag_to_metadata?(itag : JSON::Any)
- return VIDEO_FORMATS[itag.to_s]?
-end
-
def process_continuation(query, plid, id)
continuation = nil
if plid
@@ -1151,135 +359,6 @@ def process_continuation(query, plid, id)
continuation
end
-def process_video_params(query, preferences)
- annotations = query["iv_load_policy"]?.try &.to_i?
- autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- comments = query["comments"]?.try &.split(",").map(&.downcase)
- continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- player_style = query["player_style"]?
- preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
- quality = query["quality"]?
- quality_dash = query["quality_dash"]?
- region = query["region"]?
- related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- speed = query["speed"]?.try &.rchop("x").to_f?
- video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- volume = query["volume"]?.try &.to_i?
- vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
- save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
-
- if preferences
- # region ||= preferences.region
- annotations ||= preferences.annotations.to_unsafe
- autoplay ||= preferences.autoplay.to_unsafe
- comments ||= preferences.comments
- continue ||= preferences.continue.to_unsafe
- continue_autoplay ||= preferences.continue_autoplay.to_unsafe
- listen ||= preferences.listen.to_unsafe
- local ||= preferences.local.to_unsafe
- player_style ||= preferences.player_style
- preferred_captions ||= preferences.captions
- quality ||= preferences.quality
- quality_dash ||= preferences.quality_dash
- related_videos ||= preferences.related_videos.to_unsafe
- speed ||= preferences.speed
- video_loop ||= preferences.video_loop.to_unsafe
- extend_desc ||= preferences.extend_desc.to_unsafe
- volume ||= preferences.volume
- vr_mode ||= preferences.vr_mode.to_unsafe
- save_player_pos ||= preferences.save_player_pos.to_unsafe
- end
-
- annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
- autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
- comments ||= CONFIG.default_user_preferences.comments
- continue ||= CONFIG.default_user_preferences.continue.to_unsafe
- continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
- listen ||= CONFIG.default_user_preferences.listen.to_unsafe
- local ||= CONFIG.default_user_preferences.local.to_unsafe
- player_style ||= CONFIG.default_user_preferences.player_style
- preferred_captions ||= CONFIG.default_user_preferences.captions
- quality ||= CONFIG.default_user_preferences.quality
- quality_dash ||= CONFIG.default_user_preferences.quality_dash
- related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
- speed ||= CONFIG.default_user_preferences.speed
- video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
- extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
- volume ||= CONFIG.default_user_preferences.volume
- vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
- save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
-
- annotations = annotations == 1
- autoplay = autoplay == 1
- continue = continue == 1
- continue_autoplay = continue_autoplay == 1
- listen = listen == 1
- local = local == 1
- related_videos = related_videos == 1
- video_loop = video_loop == 1
- extend_desc = extend_desc == 1
- vr_mode = vr_mode == 1
- save_player_pos = save_player_pos == 1
-
- if CONFIG.disabled?("dash") && quality == "dash"
- quality = "high"
- end
-
- if CONFIG.disabled?("local") && local
- local = false
- end
-
- if start = query["t"]? || query["time_continue"]? || query["start"]?
- video_start = decode_time(start)
- end
- video_start ||= 0
-
- if query["end"]?
- video_end = decode_time(query["end"])
- end
- video_end ||= -1
-
- raw = query["raw"]?.try &.to_i?
- raw ||= 0
- raw = raw == 1
-
- controls = query["controls"]?.try &.to_i?
- controls ||= 1
- controls = controls >= 1
-
- params = VideoPreferences.new({
- annotations: annotations,
- autoplay: autoplay,
- comments: comments,
- continue: continue,
- continue_autoplay: continue_autoplay,
- controls: controls,
- listen: listen,
- local: local,
- player_style: player_style,
- preferred_captions: preferred_captions,
- quality: quality,
- quality_dash: quality_dash,
- raw: raw,
- region: region,
- related_videos: related_videos,
- speed: speed,
- video_end: video_end,
- video_loop: video_loop,
- extend_desc: extend_desc,
- video_start: video_start,
- volume: volume,
- vr_mode: vr_mode,
- save_player_pos: save_player_pos,
- })
-
- return params
-end
-
def build_thumbnails(id)
return {
{host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"},
@@ -1293,34 +372,3 @@ def build_thumbnails(id)
{host: HOST_URL, height: 90, width: 120, name: "end", url: "3"},
}
end
-
-def generate_thumbnails(json, id)
- json.array do
- build_thumbnails(id).each do |thumbnail|
- json.object do
- json.field "quality", thumbnail[:name]
- json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
- json.field "width", thumbnail[:width]
- json.field "height", thumbnail[:height]
- end
- end
- end
-end
-
-def generate_storyboards(json, id, storyboards)
- json.array do
- storyboards.each do |storyboard|
- json.object do
- json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}"
- json.field "templateUrl", storyboard[:url]
- json.field "width", storyboard[:width]
- json.field "height", storyboard[:height]
- json.field "count", storyboard[:count]
- json.field "interval", storyboard[:interval]
- json.field "storyboardWidth", storyboard[:storyboard_width]
- json.field "storyboardHeight", storyboard[:storyboard_height]
- json.field "storyboardCount", storyboard[:storyboard_count]
- end
- end
- end
-end
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
new file mode 100644
index 00000000..c811cfe1
--- /dev/null
+++ b/src/invidious/videos/caption.cr
@@ -0,0 +1,224 @@
+require "json"
+
+module Invidious::Videos
+ module Captions
+ struct Metadata
+ property name : String
+ property language_code : String
+ property base_url : String
+
+ property auto_generated : Bool
+
+ def initialize(@name, @language_code, @base_url, @auto_generated)
+ end
+
+ # Parse the JSON structure from Youtube
+ def self.from_yt_json(container : JSON::Any) : Array(Captions::Metadata)
+ caption_tracks = container
+ .dig?("playerCaptionsTracklistRenderer", "captionTracks")
+ .try &.as_a
+
+ captions_list = [] of Captions::Metadata
+ return captions_list if caption_tracks.nil?
+
+ caption_tracks.each do |caption|
+ name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
+ name = name.to_s.split(" - ")[0]
+
+ language_code = caption["languageCode"].to_s
+ base_url = caption["baseUrl"].to_s
+
+ auto_generated = (caption["kind"]? == "asr")
+
+ captions_list << Captions::Metadata.new(name, language_code, base_url, auto_generated)
+ end
+
+ return captions_list
+ end
+
+ def timedtext_to_vtt(timedtext : String, tlang = nil) : String
+ # In the future, we could just directly work with the url. This is more of a POC
+ cues = [] of XML::Node
+ tree = XML.parse(timedtext)
+ tree = tree.children.first
+
+ tree.children.each do |item|
+ if item.name == "body"
+ item.children.each do |cue|
+ if cue.name == "p" && !(cue.children.size == 1 && cue.children[0].content == "\n")
+ cues << cue
+ end
+ end
+ break
+ end
+ end
+
+ settings_field = {
+ "Kind" => "captions",
+ "Language" => "#{tlang || @language_code}",
+ }
+
+ result = WebVTT.build(settings_field) do |vtt|
+ cues.each_with_index do |node, i|
+ start_time = node["t"].to_f.milliseconds
+
+ duration = node["d"]?.try &.to_f.milliseconds
+
+ duration ||= start_time
+
+ if cues.size > i + 1
+ end_time = cues[i + 1]["t"].to_f.milliseconds
+ else
+ end_time = start_time + duration
+ end
+
+ text = String.build do |io|
+ node.children.each do |s|
+ io << s.content
+ end
+ end
+
+ vtt.cue(start_time, end_time, text)
+ end
+ end
+
+ return result
+ end
+ end
+
+ # List of all caption languages available on Youtube.
+ LANGUAGES = {
+ "",
+ "English",
+ "English (auto-generated)",
+ "English (United Kingdom)",
+ "English (United States)",
+ "Afrikaans",
+ "Albanian",
+ "Amharic",
+ "Arabic",
+ "Armenian",
+ "Azerbaijani",
+ "Bangla",
+ "Basque",
+ "Belarusian",
+ "Bosnian",
+ "Bulgarian",
+ "Burmese",
+ "Cantonese (Hong Kong)",
+ "Catalan",
+ "Cebuano",
+ "Chinese",
+ "Chinese (China)",
+ "Chinese (Hong Kong)",
+ "Chinese (Simplified)",
+ "Chinese (Taiwan)",
+ "Chinese (Traditional)",
+ "Corsican",
+ "Croatian",
+ "Czech",
+ "Danish",
+ "Dutch",
+ "Dutch (auto-generated)",
+ "Esperanto",
+ "Estonian",
+ "Filipino",
+ "Filipino (auto-generated)",
+ "Finnish",
+ "French",
+ "French (auto-generated)",
+ "Galician",
+ "Georgian",
+ "German",
+ "German (auto-generated)",
+ "Greek",
+ "Gujarati",
+ "Haitian Creole",
+ "Hausa",
+ "Hawaiian",
+ "Hebrew",
+ "Hindi",
+ "Hmong",
+ "Hungarian",
+ "Icelandic",
+ "Igbo",
+ "Indonesian",
+ "Indonesian (auto-generated)",
+ "Interlingue",
+ "Irish",
+ "Italian",
+ "Italian (auto-generated)",
+ "Japanese",
+ "Japanese (auto-generated)",
+ "Javanese",
+ "Kannada",
+ "Kazakh",
+ "Khmer",
+ "Korean",
+ "Korean (auto-generated)",
+ "Kurdish",
+ "Kyrgyz",
+ "Lao",
+ "Latin",
+ "Latvian",
+ "Lithuanian",
+ "Luxembourgish",
+ "Macedonian",
+ "Malagasy",
+ "Malay",
+ "Malayalam",
+ "Maltese",
+ "Maori",
+ "Marathi",
+ "Mongolian",
+ "Nepali",
+ "Norwegian Bokmål",
+ "Nyanja",
+ "Pashto",
+ "Persian",
+ "Polish",
+ "Portuguese",
+ "Portuguese (auto-generated)",
+ "Portuguese (Brazil)",
+ "Punjabi",
+ "Romanian",
+ "Russian",
+ "Russian (auto-generated)",
+ "Samoan",
+ "Scottish Gaelic",
+ "Serbian",
+ "Shona",
+ "Sindhi",
+ "Sinhala",
+ "Slovak",
+ "Slovenian",
+ "Somali",
+ "Southern Sotho",
+ "Spanish",
+ "Spanish (auto-generated)",
+ "Spanish (Latin America)",
+ "Spanish (Mexico)",
+ "Spanish (Spain)",
+ "Sundanese",
+ "Swahili",
+ "Swedish",
+ "Tajik",
+ "Tamil",
+ "Telugu",
+ "Thai",
+ "Turkish",
+ "Turkish (auto-generated)",
+ "Ukrainian",
+ "Urdu",
+ "Uzbek",
+ "Vietnamese",
+ "Vietnamese (auto-generated)",
+ "Welsh",
+ "Western Frisian",
+ "Xhosa",
+ "Yiddish",
+ "Yoruba",
+ "Zulu",
+ }
+ end
+end
diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr
new file mode 100644
index 00000000..29c57182
--- /dev/null
+++ b/src/invidious/videos/clip.cr
@@ -0,0 +1,22 @@
+require "json"
+
+# returns start_time, end_time and clip_title
+def parse_clip_parameters(params) : {Float64?, Float64?, String?}
+ decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
+ .try { |i| Base64.decode(i) }
+ .try { |i| IO::Memory.new(i) }
+ .try { |i| Protodec::Any.parse(i) }
+
+ start_time = decoded_protobuf
+ .try(&.["50:0:embedded"]["2:1:varint"].as_i64)
+ .try { |i| i/1000 }
+
+ end_time = decoded_protobuf
+ .try(&.["50:0:embedded"]["3:2:varint"].as_i64)
+ .try { |i| i/1000 }
+
+ clip_title = decoded_protobuf
+ .try(&.["50:0:embedded"]["4:3:string"].as_s)
+
+ return start_time, end_time, clip_title
+end
diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr
new file mode 100644
index 00000000..1371bebb
--- /dev/null
+++ b/src/invidious/videos/description.cr
@@ -0,0 +1,82 @@
+require "json"
+require "uri"
+
+private def copy_string(str : String::Builder, iter : Iterator, count : Int) : Int
+ copied = 0
+ while copied < count
+ cp = iter.next
+ break if cp.is_a?(Iterator::Stop)
+
+ if cp == 0x26 # Ampersand (&)
+ str << "&"
+ elsif cp == 0x27 # Single quote (')
+ str << "'"
+ elsif cp == 0x22 # Double quote (")
+ str << """
+ elsif cp == 0x3C # Less-than (<)
+ str << "<"
+ elsif cp == 0x3E # Greater than (>)
+ str << ">"
+ else
+ str << cp.chr
+ end
+
+ # A codepoint from the SMP counts twice
+ copied += 1 if cp > 0xFFFF
+ copied += 1
+ end
+
+ return copied
+end
+
+def parse_description(desc, video_id : String) : String?
+ return "" if desc.nil?
+
+ content = desc["content"].as_s
+ return "" if content.empty?
+
+ commands = desc["commandRuns"]?.try &.as_a
+ if commands.nil?
+ # Slightly faster than HTML.escape, as we're only doing one pass on
+ # the string instead of five for the standard library
+ return String.build do |str|
+ copy_string(str, content.each_codepoint, content.size)
+ end
+ end
+
+ # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints
+ # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are
+ # automatically decoded by the JSON parser. It means that we need to count
+ # copied byte in a special manner, preventing the use of regular string copy.
+ iter = content.each_codepoint
+
+ index = 0
+
+ return String.build do |str|
+ commands.each do |command|
+ cmd_start = command["startIndex"].as_i
+ cmd_length = command["length"].as_i
+
+ # Copy the text chunk between this command and the previous if needed.
+ length = cmd_start - index
+ index += copy_string(str, iter, length)
+
+ # We need to copy the command's text using the iterator
+ # and the special function defined above.
+ cmd_content = String.build(cmd_length) do |str2|
+ copy_string(str2, iter, cmd_length)
+ end
+
+ link = cmd_content
+ if on_tap = command.dig?("onTap", "innertubeCommand")
+ link = parse_link_endpoint(on_tap, cmd_content, video_id)
+ end
+ str << link
+ index += cmd_length
+ end
+
+ # Copy the end of the string (past the last command).
+ remaining_length = content.size - index
+ copy_string(str, iter, remaining_length) if remaining_length > 0
+ end
+end
diff --git a/src/invidious/videos/formats.cr b/src/invidious/videos/formats.cr
new file mode 100644
index 00000000..e98e7257
--- /dev/null
+++ b/src/invidious/videos/formats.cr
@@ -0,0 +1,116 @@
+module Invidious::Videos::Formats
+ def self.itag_to_metadata?(itag : JSON::Any)
+ return FORMATS[itag.to_s]?
+ end
+
+ # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
+ private FORMATS = {
+ "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
+ "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
+ "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
+ "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
+ "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
+ "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+
+ "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
+ "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+
+ # 3D videos
+ "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
+ "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
+ "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+ "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
+
+ # Apple HTTP Live Streaming
+ "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
+ "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
+ "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
+ "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
+ "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
+
+ # DASH mp4 video
+ "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
+ "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
+ "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
+ "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
+ "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
+ "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
+ "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
+ "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
+ "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
+ "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
+ "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
+ "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
+
+ # Dash mp4 audio
+ "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
+ "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
+ "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
+ "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
+ "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
+ "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
+ "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
+
+ # Dash webm
+ "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
+ "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
+ "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
+ "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
+ "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
+ "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
+ "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
+ "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
+ # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
+ "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
+ "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
+ "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+ "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
+
+ # Dash webm audio
+ "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
+ "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
+
+ # Dash webm audio with opus inside
+ "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
+ "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
+ "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
+
+ # av01 video only formats sometimes served with "unknown" codecs
+ "394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
+ "395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
+ "396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
+ "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
+ }
+end
diff --git a/src/invidious/videos/music.cr b/src/invidious/videos/music.cr
new file mode 100644
index 00000000..08d88a3e
--- /dev/null
+++ b/src/invidious/videos/music.cr
@@ -0,0 +1,13 @@
+require "json"
+
+struct VideoMusic
+ include JSON::Serializable
+
+ property song : String
+ property album : String
+ property artist : String
+ property license : String
+
+ def initialize(@song : String, @album : String, @artist : String, @license : String)
+ end
+end
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
new file mode 100644
index 00000000..feb58440
--- /dev/null
+++ b/src/invidious/videos/parser.cr
@@ -0,0 +1,504 @@
+require "json"
+
+# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
+# The former is preferred as it has more videos in it. The second has
+# the same 11 first entries as the compact rendered.
+#
+# TODO: "compactRadioRenderer" (Mix) and
+# TODO: Use a proper struct/class instead of a hacky JSON object
+def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
+ return nil if !related["videoId"]?
+
+ # The compact renderer has video length in seconds, where the end
+ # screen rendered has a full text version ("42:40")
+ length = related["lengthInSeconds"]?.try &.as_i.to_s
+ length ||= related.dig?("lengthText", "simpleText").try do |box|
+ decode_length_seconds(box.as_s).to_s
+ end
+
+ # Both have "short", so the "long" option shouldn't be required
+ channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
+ .try &.dig?("runs", 0)
+
+ author = channel_info.try &.dig?("text")
+ author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
+
+ ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
+
+ # "4,088,033 views", only available on compact renderer
+ # and when video is not a livestream
+ view_count = related.dig?("viewCountText", "simpleText")
+ .try &.as_s.gsub(/\D/, "")
+
+ short_view_count = related.try do |r|
+ HelperExtractors.get_short_view_count(r).to_s
+ end
+
+ LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
+
+ if published_time_text = related["publishedTimeText"]?
+ decoded_time = decode_date(published_time_text["simpleText"].to_s)
+ published = decoded_time.to_rfc3339.to_s
+ else
+ published = nil
+ end
+
+ # TODO: when refactoring video types, make a struct for related videos
+ # or reuse an existing type, if that fits.
+ return {
+ "id" => related["videoId"],
+ "title" => related["title"]["simpleText"],
+ "author" => author || JSON::Any.new(""),
+ "ucid" => JSON::Any.new(ucid || ""),
+ "length_seconds" => JSON::Any.new(length || "0"),
+ "view_count" => JSON::Any.new(view_count || "0"),
+ "short_view_count" => JSON::Any.new(short_view_count || "0"),
+ "author_verified" => JSON::Any.new(author_verified),
+ "published" => JSON::Any.new(published || ""),
+ }
+end
+
+def extract_video_info(video_id : String)
+ # Init client config for the API
+ client_config = YoutubeAPI::ClientConfig.new
+
+ # Fetch data from the player endpoint
+ player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
+
+ playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
+
+ if playability_status != "OK"
+ subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
+ reason = subreason.try &.[]?("simpleText").try &.as_s
+ reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
+ reason ||= player_response.dig("playabilityStatus", "reason").as_s
+
+ # Stop here if video is not a scheduled livestream or
+ # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
+ if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
+ playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
+ return {
+ "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+ "reason" => JSON::Any.new(reason),
+ }
+ end
+ elsif video_id != player_response.dig?("videoDetails", "videoId")
+ # YouTube may return a different video player response than expected.
+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+ # Line to be reverted if one day we solve the video not available issue.
+
+ # Although technically not a call to /videoplayback the fact that YouTube is returning the
+ # wrong video means that we should count it as a failure.
+ get_playback_statistic()["totalRequests"] += 1
+
+ return {
+ "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+ "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."),
+ }
+ else
+ reason = nil
+ end
+
+ # Don't fetch the next endpoint if the video is unavailable.
+ if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
+ next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
+ player_response = player_response.merge(next_response)
+ end
+
+ params = parse_video_info(video_id, player_response)
+ params["reason"] = JSON::Any.new(reason) if reason
+
+ if !CONFIG.invidious_companion.present?
+ if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
+ LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
+ players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile}
+
+ players_fallback.each do |player_fallback|
+ client_config.client_type = player_fallback
+
+ next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
+
+ if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
+ streaming_data = player_response["streamingData"].as_h
+ streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
+ player_response["streamingData"] = JSON::Any.new(streaming_data)
+ break
+ end
+ rescue InfoException
+ next LOGGER.warn("Failed to fetch streams with #{player_fallback}")
+ end
+ end
+
+ # Seems like video page can still render even without playable streams.
+ # its better than nothing.
+ #
+ # # Were we able to find playable video streams?
+ # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
+ # # No :(
+ # end
+ end
+
+ {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
+ params[f] = player_response[f] if player_response[f]?
+ end
+
+ # Convert URLs, if those are present
+ if streaming_data = player_response["streamingData"]?
+ %w[formats adaptiveFormats].each do |key|
+ streaming_data.as_h[key]?.try &.as_a.each do |format|
+ format.as_h["url"] = JSON::Any.new(convert_url(format))
+ end
+ end
+
+ params["streamingData"] = streaming_data
+ end
+
+ # Data structure version, for cache control
+ params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
+
+ return params
+end
+
+def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
+ LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
+ response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
+
+ playability_status = response["playabilityStatus"]["status"]
+ LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
+
+ if id != response.dig?("videoDetails", "videoId")
+ # YouTube may return a different video player response than expected.
+ # See: https://github.com/TeamNewPipe/NewPipe/issues/8713
+ raise InfoException.new(
+ "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
+ )
+ elsif playability_status == "OK"
+ return response
+ else
+ return nil
+ end
+end
+
+def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
+ # Top level elements
+
+ main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
+
+ raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
+
+ # Primary results are not available on Music videos
+ # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725
+ if primary_results = main_results.dig?("results", "results", "contents")
+ video_primary_renderer = primary_results
+ .as_a.find(&.["videoPrimaryInfoRenderer"]?)
+ .try &.["videoPrimaryInfoRenderer"]
+
+ video_secondary_renderer = primary_results
+ .as_a.find(&.["videoSecondaryInfoRenderer"]?)
+ .try &.["videoSecondaryInfoRenderer"]
+
+ raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
+ raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
+ end
+
+ video_details = player_response.dig?("videoDetails")
+ if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
+ microformat = {} of String => JSON::Any
+ end
+
+ raise BrokenTubeException.new("videoDetails") if !video_details
+
+ # Basic video infos
+
+ title = video_details["title"]?.try &.as_s
+
+ # We have to try to extract viewCount from videoPrimaryInfoRenderer first,
+ # then from videoDetails, as the latter is "0" for livestreams (we want
+ # to get the amount of viewers watching).
+ views_txt = extract_text(
+ video_primary_renderer
+ .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount")
+ )
+ views_txt ||= video_details["viewCount"]?.try &.as_s || ""
+ views = views_txt.gsub(/\D/, "").to_i64?
+
+ length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
+ .try &.as_s.to_i64
+
+ published = microformat["publishDate"]?
+ .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
+
+ premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
+ .try { |t| Time.parse_rfc3339(t.as_s) }
+
+ premiere_timestamp ||= player_response.dig?(
+ "playabilityStatus", "liveStreamability",
+ "liveStreamabilityRenderer", "offlineSlate",
+ "liveStreamOfflineSlateRenderer", "scheduledStartTime"
+ )
+ .try &.as_s.to_i64
+ .try { |t| Time.unix(t) }
+
+ live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
+ .try &.as_bool
+ live_now ||= video_details.dig?("isLive").try &.as_bool || false
+
+ post_live_dvr = video_details.dig?("isPostLiveDvr")
+ .try &.as_bool || false
+
+ # Extra video infos
+
+ allowed_regions = microformat["availableCountries"]?
+ .try &.as_a.map &.as_s || [] of String
+
+ allow_ratings = video_details["allowRatings"]?.try &.as_bool
+ family_friendly = microformat["isFamilySafe"]?.try &.as_bool
+ is_listed = video_details["isCrawlable"]?.try &.as_bool
+ is_upcoming = video_details["isUpcoming"]?.try &.as_bool
+
+ keywords = video_details["keywords"]?
+ .try &.as_a.map &.as_s || [] of String
+
+ # Related videos
+
+ LOGGER.debug("extract_video_info: parsing related videos...")
+
+ related = [] of JSON::Any
+
+ # Parse "compactVideoRenderer" items (under secondary results)
+ secondary_results = main_results
+ .dig?("secondaryResults", "secondaryResults", "results")
+ secondary_results.try &.as_a.each do |element|
+ if item = element["compactVideoRenderer"]?
+ related_video = parse_related_video(item)
+ related << JSON::Any.new(related_video) if related_video
+ end
+ end
+
+ # If nothing was found previously, fall back to end screen renderer
+ if related.empty?
+ # Container for "endScreenVideoRenderer" items
+ player_overlays = player_response.dig?(
+ "playerOverlays", "playerOverlayRenderer",
+ "endScreen", "watchNextEndScreenRenderer", "results"
+ )
+
+ player_overlays.try &.as_a.each do |element|
+ if item = element["endScreenVideoRenderer"]?
+ related_video = parse_related_video(item)
+ related << JSON::Any.new(related_video) if related_video
+ end
+ end
+ end
+
+ # Likes
+
+ toplevel_buttons = video_primary_renderer
+ .try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
+
+ if toplevel_buttons
+ # New Format as of december 2023
+ likes_button = toplevel_buttons.dig?(0,
+ "segmentedLikeDislikeButtonViewModel",
+ "likeButtonViewModel",
+ "likeButtonViewModel",
+ "toggleButtonViewModel",
+ "toggleButtonViewModel",
+ "defaultButtonViewModel",
+ "buttonViewModel"
+ )
+
+ likes_button ||= toplevel_buttons.try &.as_a
+ .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE")
+ .try &.["toggleButtonRenderer"]
+
+ # New format as of september 2022
+ likes_button ||= toplevel_buttons.try &.as_a
+ .find(&.["segmentedLikeDislikeButtonRenderer"]?)
+ .try &.dig?(
+ "segmentedLikeDislikeButtonRenderer",
+ "likeButton", "toggleButtonRenderer"
+ )
+
+ if likes_button
+ likes_txt = likes_button.dig?("accessibilityText")
+ # Note: The like count from `toggledText` is off by one, as it would
+ # represent the new like count in the event where the user clicks on "like".
+ likes_txt ||= (likes_button["defaultText"]? || likes_button["toggledText"]?)
+ .try &.dig?("accessibility", "accessibilityData", "label")
+ likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
+
+ LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
+ LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
+ end
+ end
+
+ # Description
+
+ description = microformat.dig?("description", "simpleText").try &.as_s || ""
+ short_description = player_response.dig?("videoDetails", "shortDescription")
+
+ # description_html = video_secondary_renderer.try &.dig?("description", "runs")
+ # .try &.as_a.try { |t| content_to_comment_html(t, video_id) }
+
+ description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"), video_id)
+
+ # Video metadata
+
+ metadata = video_secondary_renderer
+ .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
+ .try &.as_a
+
+ genre = microformat["category"]?
+ genre_ucid = nil
+ license = nil
+
+ metadata.try &.each do |row|
+ metadata_title = extract_text(row.dig?("metadataRowRenderer", "title"))
+ contents = row.dig?("metadataRowRenderer", "contents", 0)
+
+ if metadata_title == "Category"
+ contents = contents.try &.dig?("runs", 0)
+
+ genre = contents.try &.["text"]?
+ genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId")
+ elsif metadata_title == "License"
+ license = contents.try &.dig?("runs", 0, "text")
+ elsif metadata_title == "Licensed to YouTube by"
+ license = contents.try &.["simpleText"]?
+ end
+ end
+
+ # Music section
+
+ music_list = [] of VideoMusic
+ music_desclist = player_response.dig?(
+ "engagementPanels", 1, "engagementPanelSectionListRenderer",
+ "content", "structuredDescriptionContentRenderer", "items", 2,
+ "videoDescriptionMusicSectionRenderer", "carouselLockups"
+ )
+
+ music_desclist.try &.as_a.each do |music_desc|
+ artist = nil
+ album = nil
+ music_license = nil
+
+ # Used when the video has multiple songs
+ if song_title = music_desc.dig?("carouselLockupRenderer", "videoLockup", "compactVideoRenderer", "title")
+ # "simpleText" for plain text / "runs" when song has a link
+ song = song_title["simpleText"]? || song_title.dig?("runs", 0, "text")
+
+ # some videos can have empty tracks. See: https://www.youtube.com/watch?v=eBGIQ7ZuuiU
+ next if !song
+ end
+
+ music_desc.dig?("carouselLockupRenderer", "infoRows").try &.as_a.each do |desc|
+ desc_title = extract_text(desc.dig?("infoRowRenderer", "title"))
+ if desc_title == "ARTIST"
+ artist = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
+ elsif desc_title == "SONG"
+ song = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
+ elsif desc_title == "ALBUM"
+ album = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata"))
+ elsif desc_title == "LICENSES"
+ music_license = extract_text(desc.dig?("infoRowRenderer", "expandedMetadata"))
+ end
+ end
+ music_list << VideoMusic.new(song.to_s, album.to_s, artist.to_s, music_license.to_s)
+ end
+
+ # Author infos
+
+ author = video_details["author"]?.try &.as_s
+ ucid = video_details["channelId"]?.try &.as_s
+
+ if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
+ author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
+ author_verified = has_verified_badge?(author_info["badges"]?)
+
+ subs_text = author_info["subscriberCountText"]?
+ .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
+ .try &.as_s.split(" ", 2)[0]
+ end
+
+ # Return data
+
+ if live_now
+ video_type = VideoType::Livestream
+ elsif !premiere_timestamp.nil?
+ video_type = VideoType::Scheduled
+ published = premiere_timestamp || Time.utc
+ else
+ video_type = VideoType::Video
+ end
+
+ params = {
+ "videoType" => JSON::Any.new(video_type.to_s),
+ # Basic video infos
+ "title" => JSON::Any.new(title || ""),
+ "views" => JSON::Any.new(views || 0_i64),
+ "likes" => JSON::Any.new(likes || 0_i64),
+ "lengthSeconds" => JSON::Any.new(length_txt || 0_i64),
+ "published" => JSON::Any.new(published.to_rfc3339),
+ # Extra video infos
+ "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
+ "allowRatings" => JSON::Any.new(allow_ratings || false),
+ "isFamilyFriendly" => JSON::Any.new(family_friendly || false),
+ "isListed" => JSON::Any.new(is_listed || false),
+ "isUpcoming" => JSON::Any.new(is_upcoming || false),
+ "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
+ "isPostLiveDvr" => JSON::Any.new(post_live_dvr),
+ # Related videos
+ "relatedVideos" => JSON::Any.new(related),
+ # Description
+ "description" => JSON::Any.new(description || ""),
+ "descriptionHtml" => JSON::Any.new(description_html || ""),
+ "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+ # Video metadata
+ "genre" => JSON::Any.new(genre.try &.as_s || ""),
+ "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
+ "license" => JSON::Any.new(license.try &.as_s || ""),
+ # Music section
+ "music" => JSON.parse(music_list.to_json),
+ # Author infos
+ "author" => JSON::Any.new(author || ""),
+ "ucid" => JSON::Any.new(ucid || ""),
+ "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
+ "authorVerified" => JSON::Any.new(author_verified || false),
+ "subCountText" => JSON::Any.new(subs_text || "-"),
+ }
+
+ return params
+end
+
+private def convert_url(fmt)
+ if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
+ sp = cfr["sp"]
+ url = URI.parse(cfr["url"])
+ params = url.query_params
+
+ LOGGER.debug("convert_url: Decoding '#{cfr}'")
+
+ unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
+ params[sp] = unsig if unsig
+ else
+ url = URI.parse(fmt["url"].as_s)
+ params = url.query_params
+ end
+
+ n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
+ params["n"] = n if n
+
+ if token = CONFIG.po_token
+ params["pot"] = token
+ end
+
+ url.query_params = params
+ LOGGER.trace("convert_url: new url is '#{url}'")
+
+ return url.to_s
+rescue ex
+ LOGGER.debug("convert_url: Error when parsing video URL")
+ LOGGER.trace(ex.inspect_with_backtrace)
+ return ""
+end
diff --git a/src/invidious/videos/regions.cr b/src/invidious/videos/regions.cr
new file mode 100644
index 00000000..575f8c25
--- /dev/null
+++ b/src/invidious/videos/regions.cr
@@ -0,0 +1,27 @@
+# List of geographical regions that Youtube recognizes.
+# This is used to determine if a video is either restricted to a list
+# of allowed regions (= whitelisted) or if it can't be watched in
+# a set of regions (= blacklisted).
+REGIONS = {
+ "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT",
+ "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI",
+ "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY",
+ "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN",
+ "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM",
+ "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK",
+ "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL",
+ "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM",
+ "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR",
+ "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN",
+ "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS",
+ "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK",
+ "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW",
+ "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP",
+ "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM",
+ "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW",
+ "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM",
+ "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF",
+ "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW",
+ "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI",
+ "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW",
+}
diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr
new file mode 100644
index 00000000..bd0eef59
--- /dev/null
+++ b/src/invidious/videos/storyboard.cr
@@ -0,0 +1,122 @@
+require "uri"
+require "http/params"
+
+module Invidious::Videos
+ struct Storyboard
+ # Template URL
+ getter url : URI
+ getter proxied_url : URI
+
+ # Thumbnail parameters
+ getter width : Int32
+ getter height : Int32
+ getter count : Int32
+ getter interval : Int32
+
+ # Image (storyboard) parameters
+ getter rows : Int32
+ getter columns : Int32
+ getter images_count : Int32
+
+ def initialize(
+ *, @url, @width, @height, @count, @interval,
+ @rows, @columns, @images_count,
+ )
+ authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]?
+
+ @proxied_url = URI.parse(HOST_URL)
+ @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}"
+ @proxied_url.query = @url.query
+ end
+
+ # Parse the JSON structure from Youtube
+ def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard)
+ # Livestream storyboards are a bit different
+ # TODO: document exactly how
+ if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s
+ return [Storyboard.new(
+ url: URI.parse(storyboard.split("#")[0]),
+ width: 106,
+ height: 60,
+ count: -1,
+ interval: 5000,
+ rows: 3,
+ columns: 3,
+ images_count: -1
+ )]
+ end
+
+ # Split the storyboard string into chunks
+ #
+ # General format (whitespaces added for legibility):
+ # https://i.ytimg.com/sb//storyboard3_L$L/$N.jpg?sqp=
+ # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$
+ # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$
+ # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$
+ #
+ storyboards = container.dig?("playerStoryboardSpecRenderer", "spec")
+ .try &.as_s.split("|")
+
+ return [] of Storyboard if !storyboards
+
+ # The base URL is the first chunk
+ base_url = URI.parse(storyboards.shift)
+
+ return storyboards.map_with_index do |sb, i|
+ # Separate the different storyboard parameters:
+ # width/height: respective dimensions, in pixels, of a single thumbnail
+ # count: how many thumbnails are displayed across the full video
+ # columns/rows: maximum amount of thumbnails that can be stuffed in a
+ # single image, horizontally and vertically.
+ # interval: interval between two thumbnails, in milliseconds
+ # name: storyboard filename. Usually "M$M" or "default"
+ # sigh: URL cryptographic signature
+ width, height, count, columns, rows, interval, name, sigh = sb.split("#")
+
+ width = width.to_i
+ height = height.to_i
+ count = count.to_i
+ interval = interval.to_i
+ columns = columns.to_i
+ rows = rows.to_i
+
+ # Copy base URL object, so that we can modify it
+ url = base_url.dup
+
+ # Add the signature to the URL
+ params = url.query_params
+ params["sigh"] = sigh
+ url.query_params = params
+
+ # Replace the template parts with what we have
+ url.path = url.path.sub("$L", i).sub("$N", name)
+
+ # This value represents the maximum amount of thumbnails that can fit
+ # in a single image. The last image (or the only one for short videos)
+ # will contain less thumbnails than that.
+ thumbnails_per_image = columns * rows
+
+ # This value represents the total amount of storyboards required to
+ # hold all of the thumbnails. It can't be less than 1.
+ images_count = (count / thumbnails_per_image).ceil.to_i
+
+ # Compute the interval when needed (in general, that's only required
+ # for the first "default" storyboard).
+ if interval == 0
+ interval = ((length_seconds / count) * 1_000).to_i
+ end
+
+ Storyboard.new(
+ url: url,
+ width: width,
+ height: height,
+ count: count,
+ interval: interval,
+ rows: rows,
+ columns: columns,
+ images_count: images_count,
+ )
+ end
+ end
+ end
+end
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
new file mode 100644
index 00000000..ee1272d1
--- /dev/null
+++ b/src/invidious/videos/transcript.cr
@@ -0,0 +1,161 @@
+module Invidious::Videos
+ # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video.
+ # These lines can be categorized into two types: section headings and regular lines representing content from the video.
+ struct Transcript
+ # Types
+ record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+ record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+ alias TranscriptLine = HeadingLine | RegularLine
+
+ property lines : Array(TranscriptLine)
+
+ property language_code : String
+ property auto_generated : Bool
+
+ # User friendly label for the current transcript.
+ # Example: "English (auto-generated)"
+ property label : String
+
+ # Initializes a new Transcript struct with the contents and associated metadata describing it
+ def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String)
+ end
+
+ # Generates a protobuf string to fetch the requested transcript from YouTube
+ def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
+ kind = auto_generated ? "asr" : ""
+
+ object = {
+ "1:0:string" => video_id,
+
+ "2:base64" => {
+ "1:string" => kind,
+ "2:string" => language_code,
+ "3:string" => "",
+ },
+
+ "3:varint" => 1_i64,
+ "5:string" => "engagement-panel-searchable-transcript-search-panel",
+ "6:varint" => 1_i64,
+ "7:varint" => 1_i64,
+ "8:varint" => 1_i64,
+ }
+
+ params = object.try { |i| Protodec::Any.cast_json(i) }
+ .try { |i| Protodec::Any.from_json(i) }
+ .try { |i| Base64.urlsafe_encode(i) }
+ .try { |i| URI.encode_www_form(i) }
+
+ return params
+ end
+
+ # Constructs a Transcripts struct from the initial YouTube response
+ def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool)
+ transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
+ "content", "transcriptSearchPanelRenderer")
+
+ segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer")
+
+ if !segment_list["initialSegments"]?
+ raise NotFoundException.new("Requested transcript does not exist")
+ end
+
+ # Extract user-friendly label for the current transcript
+
+ footer_language_menu = transcript_panel.dig?(
+ "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems"
+ )
+
+ if footer_language_menu
+ label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
+ else
+ label = language_code
+ end
+
+ # Extract transcript lines
+
+ initial_segments = segment_list["initialSegments"].as_a
+
+ lines = [] of TranscriptLine
+
+ initial_segments.each do |line|
+ if unpacked_line = line["transcriptSectionHeaderRenderer"]?
+ line_type = HeadingLine
+ else
+ unpacked_line = line["transcriptSegmentRenderer"]
+ line_type = RegularLine
+ end
+
+ start_ms = unpacked_line["startMs"].as_s.to_i.millisecond
+ end_ms = unpacked_line["endMs"].as_s.to_i.millisecond
+ text = extract_text(unpacked_line["snippet"]) || ""
+
+ lines << line_type.new(start_ms, end_ms, text)
+ end
+
+ return Transcript.new(
+ lines: lines,
+ language_code: language_code,
+ auto_generated: auto_generated,
+ label: label
+ )
+ end
+
+ # Converts transcript lines to a WebVTT file
+ #
+ # This is used within Invidious to replace subtitles
+ # as to workaround YouTube's rate-limited timedtext endpoint.
+ def to_vtt
+ settings_field = {
+ "Kind" => "captions",
+ "Language" => @language_code,
+ }
+
+ vtt = WebVTT.build(settings_field) do |builder|
+ @lines.each do |line|
+ # Section headers are excluded from the VTT conversion as to
+ # match the regular captions returned from YouTube as much as possible
+ next if line.is_a? HeadingLine
+
+ builder.cue(line.start_ms, line.end_ms, line.line)
+ end
+ end
+
+ return vtt
+ end
+
+ def to_json(json : JSON::Builder)
+ json.field "languageCode", @language_code
+ json.field "autoGenerated", @auto_generated
+ json.field "label", @label
+ json.field "body" do
+ json.array do
+ @lines.each do |line|
+ json.object do
+ if line.is_a? HeadingLine
+ json.field "type", "heading"
+ else
+ json.field "type", "regular"
+ end
+
+ json.field "startMs", line.start_ms.total_milliseconds
+ json.field "endMs", line.end_ms.total_milliseconds
+ json.field "line", line.line
+ end
+ end
+ end
+ end
+ end
+
+ def to_json
+ JSON.build do |json|
+ json.object do
+ json.field "transcript" do
+ json.object do
+ to_json(json)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr
new file mode 100644
index 00000000..48177bd8
--- /dev/null
+++ b/src/invidious/videos/video_preferences.cr
@@ -0,0 +1,162 @@
+struct VideoPreferences
+ include JSON::Serializable
+
+ property annotations : Bool
+ property preload : Bool
+ property autoplay : Bool
+ property comments : Array(String)
+ property continue : Bool
+ property continue_autoplay : Bool
+ property controls : Bool
+ property listen : Bool
+ property local : Bool
+ property preferred_captions : Array(String)
+ property player_style : String
+ property quality : String
+ property quality_dash : String
+ property raw : Bool
+ property region : String?
+ property related_videos : Bool
+ property speed : Float32 | Float64
+ property video_end : Float64 | Int32
+ property video_loop : Bool
+ property extend_desc : Bool
+ property video_start : Float64 | Int32
+ property volume : Int32
+ property vr_mode : Bool
+ property save_player_pos : Bool
+end
+
+def process_video_params(query, preferences)
+ annotations = query["iv_load_policy"]?.try &.to_i?
+ preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ comments = query["comments"]?.try &.split(",").map(&.downcase)
+ continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ player_style = query["player_style"]?
+ preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
+ quality = query["quality"]?
+ quality_dash = query["quality_dash"]?
+ region = query["region"]?
+ related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ speed = query["speed"]?.try &.rchop("x").to_f?
+ video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ volume = query["volume"]?.try &.to_i?
+ vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+ save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
+
+ if preferences
+ # region ||= preferences.region
+ annotations ||= preferences.annotations.to_unsafe
+ preload ||= preferences.preload.to_unsafe
+ autoplay ||= preferences.autoplay.to_unsafe
+ comments ||= preferences.comments
+ continue ||= preferences.continue.to_unsafe
+ continue_autoplay ||= preferences.continue_autoplay.to_unsafe
+ listen ||= preferences.listen.to_unsafe
+ local ||= preferences.local.to_unsafe
+ player_style ||= preferences.player_style
+ preferred_captions ||= preferences.captions
+ quality ||= preferences.quality
+ quality_dash ||= preferences.quality_dash
+ related_videos ||= preferences.related_videos.to_unsafe
+ speed ||= preferences.speed
+ video_loop ||= preferences.video_loop.to_unsafe
+ extend_desc ||= preferences.extend_desc.to_unsafe
+ volume ||= preferences.volume
+ vr_mode ||= preferences.vr_mode.to_unsafe
+ save_player_pos ||= preferences.save_player_pos.to_unsafe
+ end
+
+ annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
+ preload ||= CONFIG.default_user_preferences.preload.to_unsafe
+ autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
+ comments ||= CONFIG.default_user_preferences.comments
+ continue ||= CONFIG.default_user_preferences.continue.to_unsafe
+ continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
+ listen ||= CONFIG.default_user_preferences.listen.to_unsafe
+ local ||= CONFIG.default_user_preferences.local.to_unsafe
+ player_style ||= CONFIG.default_user_preferences.player_style
+ preferred_captions ||= CONFIG.default_user_preferences.captions
+ quality ||= CONFIG.default_user_preferences.quality
+ quality_dash ||= CONFIG.default_user_preferences.quality_dash
+ related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
+ speed ||= CONFIG.default_user_preferences.speed
+ video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
+ extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
+ volume ||= CONFIG.default_user_preferences.volume
+ vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
+ save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
+
+ annotations = annotations == 1
+ preload = preload == 1
+ autoplay = autoplay == 1
+ continue = continue == 1
+ continue_autoplay = continue_autoplay == 1
+ listen = listen == 1
+ local = local == 1
+ related_videos = related_videos == 1
+ video_loop = video_loop == 1
+ extend_desc = extend_desc == 1
+ vr_mode = vr_mode == 1
+ save_player_pos = save_player_pos == 1
+
+ if CONFIG.disabled?("dash") && quality == "dash"
+ quality = "high"
+ end
+
+ if CONFIG.disabled?("local") && local
+ local = false
+ end
+
+ if start = query["t"]? || query["time_continue"]? || query["start"]?
+ video_start = decode_time(start)
+ end
+ video_start ||= 0
+
+ if query["end"]?
+ video_end = decode_time(query["end"])
+ end
+ video_end ||= -1
+
+ raw = query["raw"]?.try &.to_i?
+ raw ||= 0
+ raw = raw == 1
+
+ controls = query["controls"]?.try &.to_i?
+ controls ||= 1
+ controls = controls >= 1
+
+ params = VideoPreferences.new({
+ annotations: annotations,
+ preload: preload,
+ autoplay: autoplay,
+ comments: comments,
+ continue: continue,
+ continue_autoplay: continue_autoplay,
+ controls: controls,
+ listen: listen,
+ local: local,
+ player_style: player_style,
+ preferred_captions: preferred_captions,
+ quality: quality,
+ quality_dash: quality_dash,
+ raw: raw,
+ region: region,
+ related_videos: related_videos,
+ speed: speed,
+ video_end: video_end,
+ video_loop: video_loop,
+ extend_desc: extend_desc,
+ video_start: video_start,
+ volume: volume,
+ vr_mode: vr_mode,
+ save_player_pos: save_player_pos,
+ })
+
+ return params
+end
diff --git a/src/invidious/views/add_playlist_items.ecr b/src/invidious/views/add_playlist_items.ecr
index ad50909a..6aea82ae 100644
--- a/src/invidious/views/add_playlist_items.ecr
+++ b/src/invidious/views/add_playlist_items.ecr
@@ -11,7 +11,9 @@
@@ -29,30 +31,5 @@
-
- <% videos.each_slice(4) do |slice| %>
- <% slice.each do |item| %>
- <%= rendered "components/item" %>
- <% end %>
- <% end %>
-
-<% if query %>
-
-
- <% if page > 1 %>
-
- <%= translate(locale, "Previous page") %>
-
- <% end %>
-
-
-
- <% if videos.size >= 20 %>
-
- <%= translate(locale, "Next page") %>
-
- <% end %>
-
-
-<% end %>
+<%= rendered "components/items_paginated" %>
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 40b553a9..686de6bd 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,116 +1,59 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url =
+ case selected_tab
+ when .shorts? then "/channel/#{ucid}/shorts"
+ when .streams? then "/channel/#{ucid}/streams"
+ when .playlists? then "/channel/#{ucid}/playlists"
+ when .channels? then "/channel/#{ucid}/channels"
+ when .podcasts? then "/channel/#{ucid}/podcasts"
+ when .releases? then "/channel/#{ucid}/releases"
+ when .courses? then "/channel/#{ucid}/courses"
+ else
+ "/channel/#{ucid}"
+ end
+
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
+ base_url: relative_url,
+ ctoken: next_continuation,
+ first_page: continuation.nil?,
+ params: env.params.query,
+ )
+%>
<% content_for "header" do %>
-<%= author %> - Invidious
+<%- if selected_tab.videos? -%>
+
+
+
+
+
+
+
+
+
+
+
+<%- end -%>
+
+
+
+
+<%= author %> - Invidious
<% end %>
-<% if channel.banner %>
-
- ">
-
-
-
-
-
-<% end %>
-
-
-
-
-
- <%= channel.description_html %>
-
-
-
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
- <%= translate(locale, "View channel on YouTube") %>
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
- <% if !channel.auto_generated %>
-
- <%= translate(locale, "Videos") %>
-
- <% end %>
-
- <% if channel.auto_generated %>
- <%= translate(locale, "Playlists") %>
- <% else %>
- <%= translate(locale, "Playlists") %>
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
-
-
-
-
- <% sort_options.each do |sort| %>
-
- <% if sort_by == sort %>
- <%= translate(locale, sort) %>
- <% else %>
-
- <%= translate(locale, sort) %>
-
- <% end %>
-
- <% end %>
-
-
-
+<%= rendered "components/channel_info" %>
-
-<% items.each do |item| %>
- <%= rendered "components/item" %>
-<% end %>
-
-
-
- <% if page > 1 %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Previous page") %>
-
- <% end %>
-
-
-
- <% if count == 60 %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Next page") %>
-
- <% end %>
-
-
+<%= rendered "components/items_paginated" %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index f0add06b..132e636c 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,71 +1,21 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url = "/channel/#{ucid}/community"
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Posts
+-%>
<% content_for "header" do %>
+
<%= author %> - Invidious
<% end %>
-<% if channel.banner %>
-
- ">
-
-
-
-
-
-<% end %>
-
-
-
-
-
- <%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>
-
-
-
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
- <%= translate(locale, "View channel on YouTube") %>
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
- <% if !channel.auto_generated %>
-
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
-
-
-
+<%= rendered "components/channel_info" %>
@@ -76,8 +26,8 @@
<%= error_message %>
<% else %>
-
- <%= template_youtube_comments(items.not_nil!, locale, thin_mode) %>
+
+ <%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode) %>
<% end %>
diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr
new file mode 100644
index 00000000..f4164f31
--- /dev/null
+++ b/src/invidious/views/components/channel_info.ecr
@@ -0,0 +1,61 @@
+<% if channel.banner %>
+
+ " alt="" />
+
+
+
+
+
+<% end %>
+
+
+
+
+
+ <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %>
+
+
+
+
+
+
+
+ <%= channel.description_html %>
+
+
+
+
+
+
+
+ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
+
+
+
+ <% sort_options.each do |sort| %>
+
+ <% if sort_by == sort %>
+ <%= translate(locale, sort) %>
+ <% else %>
+ <%= translate(locale, sort) %>
+ <% end %>
+
+ <% end %>
+
+
+
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr
index 5f8bde13..a24423df 100644
--- a/src/invidious/views/components/item.ecr
+++ b/src/invidious/views/components/item.ecr
@@ -1,151 +1,199 @@
+<%-
+ thin_mode = env.get("preferences").as(Preferences).thin_mode
+ item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category | ProblematicTimelineItem) && env.get?("user").try &.as(User).watched.index(item.id) != nil
+ author_verified = item.responds_to?(:author_verified) && item.author_verified
+-%>
+
<% case item when %>
<% when SearchChannel %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
+ <% if !thin_mode %>
+
- "/>
+ " alt="" />
- <% end %>
- <%= HTML.escape(item.author) %>
-
- <%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>
- <% if !item.auto_generated %><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
<% end %>
- <%= item.description_html %>
- <% when SearchPlaylist, InvidiousPlaylist %>
- <% if item.id.starts_with? "RD" %>
- <% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %>
- <% else %>
- <% url = "/playlist?list=#{item.id}" %>
+
+ <%- else -%>
+
<% end %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
- "/>
- <%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
-
- <% end %>
- <%= HTML.escape(item.title) %>
-
-
- <%= HTML.escape(item.author) %>
-
- <% when MixVideo %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
-
- <% if item.length_seconds != 0 %>
- <%= recode_length_seconds(item.length_seconds) %>
- <% end %>
-
- <% end %>
- <%= HTML.escape(item.title) %>
-
-
- <%= HTML.escape(item.author) %>
-
- <% when PlaylistVideo %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
-
- <% if plid = env.get?("remove_playlist_items") %>
-
- <% end %>
-
- <% if item.responds_to?(:live_now) && item.live_now %>
- <%= translate(locale, "LIVE") %>
- <% elsif item.length_seconds != 0 %>
- <%= recode_length_seconds(item.length_seconds) %>
- <% end %>
-
- <% end %>
- <%= HTML.escape(item.title) %>
-
-
- <% endpoint_params = "?v=#{item.id}&list=#{item.plid}" %>
- <%= rendered "components/video-context-buttons" %>
+
+
+ <% if !item.channel_handle.nil? %><%= item.channel_handle %>
<% end %>
+ <%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>
+ <% if !item.auto_generated && item.channel_handle.nil? %><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
<% end %>
+ <%= item.description_html %>
+ <% when SearchHashtag %>
+ <% if !thin_mode %>
+
+ <%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
+ <%- end -%>
+
+
+
+ <%- if item.channel_count != 0 -%>
+ <%= translate_count(locale, "generic_channels_count", item.channel_count, NumberFormatting::Separator) %>
+ <%- end -%>
+
+ <% when SearchPlaylist, InvidiousPlaylist %>
+ <%-
+ if item.id.starts_with? "RD"
+ link_url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}"
+ else
+ link_url = "/playlist?list=#{item.id}"
+ end
+ -%>
+
+
+ <%- if !thin_mode %>
+
+ " alt="" />
+
+ <%- else -%>
+
+ <%- end -%>
+
+
+
+
+
- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
- <%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
- <% elsif Time.utc - item.published > 1.minute %>
- <%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
+ <% if !item.ucid.to_s.empty? %>
+
+ <%= HTML.escape(item.author) %>
+ <%- if author_verified %> <% end -%>
+
+
+ <% else %>
+ <%= HTML.escape(item.author) %>
+ <%- if author_verified %> <% end -%>
+
+ <% end %>
+
+
+ <% when Category %>
+ <% when ProblematicTimelineItem %>
+
+
+
+ <%=translate(locale, "timeline_parse_error_placeholder_heading")%>
+ <%=translate(locale, "timeline_parse_error_placeholder_message")%>
+
+
+ <%=translate(locale, "timeline_parse_error_show_technical_details")%>
+ <%=get_issue_template(env, item.parse_exception)[1]%>
+
+
+ <% else %>
+ <%-
+ # `endpoint_params` is used for the "video-context-buttons" component
+ if item.is_a?(PlaylistVideo)
+ link_url = "/watch?v=#{item.id}&list=#{item.plid}&index=#{item.index}"
+ endpoint_params = "?v=#{item.id}&list=#{item.plid}"
+ elsif item.is_a?(MixVideo)
+ link_url = "/watch?v=#{item.id}&list=#{item.rdid}"
+ endpoint_params = "?v=#{item.id}&list=#{item.rdid}"
+ else
+ link_url = "/watch?v=#{item.id}"
+ endpoint_params = "?v=#{item.id}"
+ end
+ -%>
+
+
+ <%- if !thin_mode -%>
+
+
+
+ <% if item_watched %>
+
+
+ <% end %>
+
+ <%- else -%>
+
+ <%- end -%>
+
+
+
+
+
+
+
+
+
+
+ <% if !item.ucid.to_s.empty? %>
+
+ <%= HTML.escape(item.author) %>
+ <%- if author_verified %> <% end -%>
+
+
+ <% else %>
+ <%= HTML.escape(item.author) %>
+ <%- if author_verified %> <% end -%>
+
<% end %>
- <% if item.responds_to?(:views) && item.views %>
-
- <%= translate_count(locale, "generic_views_count", item.views || 0, NumberFormatting::Short) %>
-
- <% end %>
-
- <% when Category %>
- <% else %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
-
- <% if env.get? "show_watched" %>
-
- <% elsif plid = env.get? "add_playlist_items" %>
-
- <% end %>
-
- <% if item.responds_to?(:live_now) && item.live_now %>
- <%= translate(locale, "LIVE") %>
- <% elsif item.length_seconds != 0 %>
- <%= recode_length_seconds(item.length_seconds) %>
- <% end %>
-
- <% end %>
- <%= HTML.escape(item.title) %>
-
-
-
-
-
- <% endpoint_params = "?v=#{item.id}" %>
<%= rendered "components/video-context-buttons" %>
@@ -153,7 +201,7 @@
<% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
- <% elsif Time.utc - item.published > 1.minute %>
+ <% elsif item.responds_to?(:published) && (Time.utc - item.published) > 1.minute %>
<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% end %>
diff --git a/src/invidious/views/components/items_paginated.ecr b/src/invidious/views/components/items_paginated.ecr
new file mode 100644
index 00000000..f69df3fe
--- /dev/null
+++ b/src/invidious/views/components/items_paginated.ecr
@@ -0,0 +1,21 @@
+<%= page_nav_html %>
+
+
+ <%- items.each do |item| -%>
+ <%= rendered "components/item" %>
+ <%- end -%>
+
+
+<%= page_nav_html %>
+
+
+
+
diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr
index 206ba380..af352102 100644
--- a/src/invidious/views/components/player.ecr
+++ b/src/invidious/views/components/player.ecr
@@ -1,18 +1,47 @@
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
#{video["title"]}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index 9bb73136..c8db207c 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -42,11 +42,6 @@ module Invidious::Routes::Account sid = sid.as(String) token = env.params.body["csrf_token"]? - # We don't store passwords for Google accounts - if !user.password - return error_template(400, "Cannot change password for Google accounts") - end - begin validate_request(token, sid, env.request, HMAC_KEY, locale) rescue ex @@ -54,11 +49,11 @@ module Invidious::Routes::Account end password = env.params.body["password"]? - if !password + if password.nil? || password.empty? return error_template(401, "Password is a required field") end - new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } + new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 return error_template(400, "New passwords must match") @@ -203,7 +198,7 @@ module Invidious::Routes::Account referer = get_referer(env) if !user - return env.redirect referer + return env.redirect "/login?referer=#{URI.encode_path_segment(env.request.resource)}" end user = user.as(User) @@ -245,7 +240,7 @@ module Invidious::Routes::Account return error_template(400, ex) end - scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } + scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? @@ -262,6 +257,7 @@ module Invidious::Routes::Account end query["token"] = access_token + query["username"] = URI.encode_path_segment(user.email) url.query = query.to_s env.redirect url.to_s @@ -332,17 +328,9 @@ module Invidious::Routes::Account end end - if env.params.query["action_revoke_token"]? - action = "action_revoke_token" - else - return env.redirect referer - end - - session = env.params.query["session"]? - session ||= "" - - case action - when .starts_with? "action_revoke_token" + case action = env.params.query["action"]? + when "revoke_token" + session = env.params.query["session"] Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index ca429df5..c27caad7 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -8,45 +8,49 @@ module Invidious::Routes::API::Manifest id = env.params.url["id"] region = env.params.query["region"]? + if CONFIG.invidious_companion.present? + invidious_companion = CONFIG.invidious_companion.sample + return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}" + end + # Since some implementations create playlists based on resolution regardless of different codecs, # we can opt to only add a source to a representation if it has a unique height within that representation unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } begin video = get_video(id, region: region) - rescue ex : VideoRedirect - return env.redirect env.request.resource.gsub(id, ex.video_id) + rescue ex : NotFoundException + haltf env, status_code: 404 rescue ex haltf env, status_code: 403 end if dashmpd = video.dash_manifest_url - manifest = YT_POOL.client &.get(URI.parse(dashmpd).request_target).body + response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) - manifest = manifest.gsub(/" - - user_hash = lookup_results[0][2] - - if token = env.params.body["token"]? - answer = env.params.body["answer"]? - captcha = {token, answer} - else - captcha = nil - end - - challenge_req = { - user_hash, nil, 1, nil, - {1, nil, nil, nil, - {password, captcha, true}, - }, - {nil, nil, - {2, 1, nil, 1, - "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn", - nil, [] of String, 4}, - 1, - {nil, nil, [] of String}, - nil, nil, nil, true, - }, - }.to_json - - traceback << "Getting challenge..." - - response = client.post("/_/signin/sl/challenge", headers, login_req(challenge_req)) - headers = response.cookies.add_request_headers(headers) - challenge_results = JSON.parse(response.body[5..-1]) - - traceback << "done, returned #{response.status_code}.
" - - headers["Cookie"] = URI.decode_www_form(headers["Cookie"]) - - if challenge_results[0][3]?.try &.== 7 - return error_template(423, "Account has temporarily been disabled") - end - - if token = challenge_results[0][-1]?.try &.[-1]?.try &.as_h?.try &.["5001"]?.try &.[-1].as_a?.try &.[-1].as_s - account_type = "google" - captcha_type = "image" - prompt = nil - tfa = tfa_code - captcha = {tokens: [token], question: ""} - - return templated "user/login" - end - - if challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED" - return error_template(401, "Incorrect password") - end - - prompt_type = challenge_results[0][-1]?.try &.[0].as_a?.try &.[0][2]? - if {"TWO_STEP_VERIFICATION", "LOGIN_CHALLENGE"}.includes? prompt_type - traceback << "Handling prompt #{prompt_type}.
" - case prompt_type - when "TWO_STEP_VERIFICATION" - prompt_type = 2 - else # "LOGIN_CHALLENGE" - prompt_type = 4 - end - - # Prefer Authenticator app and SMS over unsupported protocols - if !{6, 9, 12, 15}.includes?(challenge_results[0][-1][0][0][8].as_i) && prompt_type == 2 - tfa = challenge_results[0][-1][0].as_a.select { |auth_type| {6, 9, 12, 15}.includes? auth_type[8] }[0] - - traceback << "Selecting challenge #{tfa[8]}..." - select_challenge = {prompt_type, nil, nil, nil, {tfa[8]}}.to_json - - tl = challenge_results[1][2] - - tfa = client.post("/_/signin/selectchallenge?TL=#{tl}", headers, login_req(select_challenge)).body - tfa = tfa[5..-1] - tfa = JSON.parse(tfa)[0][-1] - - traceback << "done.
" - else - traceback << "Using challenge #{challenge_results[0][-1][0][0][8]}.
" - tfa = challenge_results[0][-1][0][0] - end - - if tfa[5] == "QUOTA_EXCEEDED" - return error_template(423, "Quota exceeded, try again in a few hours") - end - - if !tfa_code - account_type = "google" - captcha_type = "image" - - case tfa[8] - when 6, 9 - prompt = "Google verification code" - when 12 - prompt = "Login verification, recovery email: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" - when 15 - prompt = "Login verification, security question: #{tfa[-1][tfa[-1].as_h.keys[0]][0]}" - else - prompt = "Google verification code" - end - - tfa = nil - captcha = nil - return templated "user/login" - end - - tl = challenge_results[1][2] - - request_type = tfa[8] - case request_type - when 6 # Authenticator app - tfa_req = { - user_hash, nil, 2, nil, - {6, nil, nil, nil, nil, - {tfa_code, false}, - }, - }.to_json - when 9 # Voice or text message - tfa_req = { - user_hash, nil, 2, nil, - {9, nil, nil, nil, nil, nil, nil, nil, - {nil, tfa_code, false, 2}, - }, - }.to_json - when 12 # Recovery email - tfa_req = { - user_hash, nil, 4, nil, - {12, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - {tfa_code}, - }, - }.to_json - when 15 # Security question - tfa_req = { - user_hash, nil, 5, nil, - {15, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - {tfa_code}, - }, - }.to_json - else - return error_template(500, "Unable to log in, make sure two-factor authentication (Authenticator or SMS) is turned on.") - end - - traceback << "Submitting challenge..." - - response = client.post("/_/signin/challenge?hl=en&TL=#{tl}", headers, login_req(tfa_req)) - headers = response.cookies.add_request_headers(headers) - challenge_results = JSON.parse(response.body[5..-1]) - - if (challenge_results[0][-1]?.try &.[5] == "INCORRECT_ANSWER_ENTERED") || - (challenge_results[0][-1]?.try &.[5] == "INVALID_INPUT") - return error_template(401, "Invalid TFA code") - end - - traceback << "done.
" - end - - traceback << "Logging in..." - - location = URI.parse(challenge_results[0][-1][2].to_s) - cookies = HTTP::Cookies.from_client_headers(headers) - - headers.delete("Content-Type") - headers.delete("Google-Accounts-XSRF") - - loop do - if !location || location.path == "/ManageAccount" - break - end - - # Occasionally there will be a second page after login confirming - # the user's phone number ("/b/0/SmsAuthInterstitial"), which we currently don't handle. - - if location.path.starts_with? "/b/0/SmsAuthInterstitial" - traceback << "Unhandled dialog /b/0/SmsAuthInterstitial." - end - - login = client.get(location.request_target, headers) - - headers = login.cookies.add_request_headers(headers) - location = login.headers["Location"]?.try { |u| URI.parse(u) } - end - - cookies = HTTP::Cookies.from_client_headers(headers) - sid = cookies["SID"]?.try &.value - if !sid - raise "Couldn't get SID." - end - - user, sid = get_user(sid, headers) - - # We are now logged in - traceback << "done.
" - - host = URI.parse(env.request.headers["Host"]).host - - cookies.each do |cookie| - cookie.secure = Invidious::User::Cookies::SECURE - - if cookie.extension - cookie.extension = cookie.extension.not_nil!.gsub(".youtube.com", host) - cookie.extension = cookie.extension.not_nil!.gsub("Secure; ", "") - end - env.response.cookies << cookie - end - - if env.request.cookies["PREFS"]? - user.preferences = env.get("preferences").as(Preferences) - Invidious::Database::Users.update_preferences(user) - - cookie = env.request.cookies["PREFS"] - cookie.expires = Time.utc(1990, 1, 1) - env.response.cookies << cookie - end - - env.redirect referer - rescue ex - traceback.rewind - # error_message = translate(locale, "Login failed. This may be because two-factor authentication is not turned on for your account.") - error_message = %(#{ex.message}
Traceback:
#{traceback.gets_to_end}
)
- return error_template(500, error_message)
- end
when "invidious"
- if !email
+ if email.nil? || email.empty?
return error_template(401, "User ID is a required field")
end
- if !password
+ if password.nil? || password.empty?
return error_template(401, "Password is a required field")
end
user = Invidious::Database::Users.select(email: email)
if user
- if !user.password
- return error_template(400, "Please sign in using 'Log in with Google'")
- end
-
if Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
Invidious::Database::SessionIDs.insert(sid, email)
@@ -356,36 +85,14 @@ module Invidious::Routes::Login
password = password.byte_slice(0, 55)
if CONFIG.captcha_enabled
- captcha_type = env.params.body["captcha_type"]?
answer = env.params.body["answer"]?
- change_type = env.params.body["change_type"]?
- if !captcha_type || change_type
- if change_type
- captcha_type = change_type
- end
- captcha_type ||= "image"
-
- account_type = "invidious"
- tfa = false
- prompt = ""
-
- if captcha_type == "image"
- captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
- else
- captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
- end
-
- return templated "user/login"
- end
+ account_type = "invidious"
+ captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
- answer ||= ""
- captcha_type ||= "image"
-
- case captcha_type
- when "image"
+ if answer
answer = answer.lstrip('0')
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
@@ -394,27 +101,8 @@ module Invidious::Routes::Login
rescue ex
return error_template(400, ex)
end
- else # "text"
- answer = Digest::MD5.hexdigest(answer.downcase.strip)
-
- if tokens.empty?
- return error_template(500, "Erroneous CAPTCHA")
- end
-
- found_valid_captcha = false
- error_exception = Exception.new
- tokens.each do |tok|
- begin
- validate_request(tok, answer, env.request, HMAC_KEY, locale)
- found_valid_captcha = true
- rescue ex
- error_exception = ex
- end
- end
-
- if !found_valid_captcha
- return error_template(500, error_exception)
- end
+ else
+ return templated "user/login"
end
end
@@ -481,11 +169,4 @@ module Invidious::Routes::Login
env.redirect referer
end
-
- def self.captcha(env)
- headers = HTTP::Headers{":authority" => "accounts.google.com"}
- response = YT_POOL.client &.get(env.request.resource, headers)
- env.response.headers["Content-Type"] = response.headers["Content-Type"]
- response.body
- end
end
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index d6bd9571..0b868755 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -40,7 +40,21 @@ module Invidious::Routes::Misc
def self.cross_instance_redirect(env)
referer = get_referer(env)
- instance_url = fetch_random_instance
+
+ instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
+ # Filter out the current instance
+ other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
+
+ if other_available_instances.empty?
+ # If the current instance is the only one, use the redirect URL as fallback
+ instance_url = "redirect.invidious.io"
+ else
+ # Select other random instance
+ # Sample returns an array
+ # Instances are packaged as {region, domain} in the instance list
+ instance_url = other_available_instances.sample(1)[0][1]
+ end
+
env.redirect "https://#{instance_url}#{referer}"
end
end
diff --git a/src/invidious/routes/notifications.cr b/src/invidious/routes/notifications.cr
index 272a3dc7..8922b740 100644
--- a/src/invidious/routes/notifications.cr
+++ b/src/invidious/routes/notifications.cr
@@ -24,50 +24,6 @@ module Invidious::Routes::Notifications
user = user.as(User)
- if !user.password
- channel_req = {} of String => String
-
- channel_req["receive_all_updates"] = env.params.query["receive_all_updates"]? || "true"
- channel_req["receive_no_updates"] = env.params.query["receive_no_updates"]? || ""
- channel_req["receive_post_updates"] = env.params.query["receive_post_updates"]? || "true"
-
- channel_req.reject! { |k, v| v != "true" && v != "false" }
-
- headers = HTTP::Headers.new
- headers["Cookie"] = env.request.headers["Cookie"]
-
- html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
-
- cookies = HTTP::Cookies.from_client_headers(headers)
- html.cookies.each do |cookie|
- if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name
- if cookies[cookie.name]?
- cookies[cookie.name] = cookie
- else
- cookies << cookie
- end
- end
- end
- headers = cookies.add_request_headers(headers)
-
- if match = html.body.match(/'XSRF_TOKEN': "(?#{ex.message}") end @@ -421,6 +432,13 @@ module Invidious::Routes::Playlists env.set "remove_playlist_items", plid end + # Pagination + page_nav_html = Frontend::Pagination.nav_numeric(locale, + base_url: "/playlist?list=#{playlist.id}", + current_page: page, + show_next: (page_count != 1 && page < page_count) + ) + templated "playlist" end diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 570cba69..39ca77c0 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -27,6 +27,10 @@ module Invidious::Routes::PreferencesRoute annotations_subscribed ||= "off" annotations_subscribed = annotations_subscribed == "on" + preload = env.params.body["preload"]?.try &.as(String) + preload ||= "off" + preload = preload == "on" + autoplay = env.params.body["autoplay"]?.try &.as(String) autoplay ||= "off" autoplay = autoplay == "on" @@ -144,6 +148,7 @@ module Invidious::Routes::PreferencesRoute preferences = Preferences.from_json({ annotations: annotations, annotations_subscribed: annotations_subscribed, + preload: preload, autoplay: autoplay, captions: captions, comments: comments, @@ -214,7 +219,7 @@ module Invidious::Routes::PreferencesRoute statistics_enabled ||= "off" CONFIG.statistics_enabled = statistics_enabled == "on" - CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String) + CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence File.write("config/config.yml", CONFIG.to_yaml) end @@ -310,6 +315,24 @@ module Invidious::Routes::PreferencesRoute response: error_template(415, "Invalid subscription file uploaded") ) end + when "import_youtube_pl" + filename = part.filename || "" + success = Invidious::User::Import.from_youtube_pl(user, body, filename, type) + + if !success + haltf(env, status_code: 415, + response: error_template(415, "Invalid playlist file uploaded") + ) + end + when "import_youtube_wh" + filename = part.filename || "" + success = Invidious::User::Import.from_youtube_wh(user, body, filename, type) + + if !success + haltf(env, status_code: 415, + response: error_template(415, "Invalid watch history file uploaded") + ) + end when "import_freetube" Invidious::User::Import.from_freetube(user, body) when "import_newpipe_subscriptions" diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 3f4c7e5e..b195c7b3 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -37,38 +37,87 @@ module Invidious::Routes::Search end def self.search(env) - locale = env.get("preferences").as(Preferences).locale - region = env.params.query["region"]? + prefs = env.get("preferences").as(Preferences) + locale = prefs.locale - query = env.params.query["search_query"]? - query ||= env.params.query["q"]? + region = env.params.query["region"]? || prefs.region - if !query || query.empty? + query = Invidious::Search::Query.new(env.params.query, :regular, region) + + if query.empty? # Display the full page search box implemented in #1977 env.set "search", "" templated "search_homepage", navbar_search: false else - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - user = env.get? "user" + # An URL was copy/pasted in the search box. + # Redirect the user to the appropriate page. + if query.url? + return env.redirect UrlSanitizer.process(query.text).to_s + end + begin - search_query, videos, operators = process_search_query(query, page, user, region: region) + if user + items = query.process(user.as(User)) + else + items = query.process + end rescue ex : ChannelSearchException return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") rescue ex return error_template(500, ex) end - operator_hash = {} of String => String - operators.each do |operator| - key, value = operator.downcase.split(":") - operator_hash[key] = value + redirect_url = Invidious::Frontend::Misc.redirect_url(env) + + # Pagination + page_nav_html = Frontend::Pagination.nav_numeric(locale, + base_url: "/search?#{query.to_http_params}", + current_page: query.page, + show_next: (items.size >= 20) + ) + + if query.type == Invidious::Search::Query::Type::Channel + env.set "search", "channel:#{query.channel} #{query.text}" + else + env.set "search", query.text end - env.set "search", query templated "search" end end + + def self.hashtag(env : HTTP::Server::Context) + locale = env.get("preferences").as(Preferences).locale + + hashtag = env.params.url["hashtag"]? + if hashtag.nil? || hashtag.empty? + return error_template(400, "Invalid request") + end + + page = env.params.query["page"]? + if page.nil? + page = 1 + else + page = Math.max(1, page.to_i) + env.params.query.delete_all("page") + end + + begin + items = Invidious::Hashtag.fetch(hashtag, page) + rescue ex + return error_template(500, ex) + end + + # Pagination + hashtag_encoded = URI.encode_www_form(hashtag, space_to_plus: false) + page_nav_html = Frontend::Pagination.nav_numeric(locale, + base_url: "/hashtag/#{hashtag_encoded}", + current_page: page, + show_next: (items.size >= 60) + ) + + templated "hashtag" + end end diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr index 7b1fa876..1de655d2 100644 --- a/src/invidious/routes/subscriptions.cr +++ b/src/invidious/routes/subscriptions.cr @@ -32,29 +32,16 @@ module Invidious::Routes::Subscriptions end end - if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1 - action = "action_create_subscription_to_channel" - elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1 - action = "action_remove_subscriptions" - else - return env.redirect referer - end - channel_id = env.params.query["c"]? channel_id ||= "" - if !user.password - # Sync subscriptions with YouTube - subscribe_ajax(channel_id, action, env.request.headers) - end - - case action - when "action_create_subscription_to_channel" + case action = env.params.query["action"]? + when "create_subscription_to_channel" if !user.subscriptions.includes? channel_id get_channel(channel_id) Invidious::Database::Users.subscribe_channel(user, channel_id) end - when "action_remove_subscriptions" + when "remove_subscriptions" Invidious::Database::Users.unsubscribe_channel(user, channel_id) else return error_json(400, "Unsupported action #{action}") @@ -82,14 +69,6 @@ module Invidious::Routes::Subscriptions user = user.as(User) sid = sid.as(String) - if !user.password - # Refresh account - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] - - user, sid = get_user(sid, headers) - end - action_takeout = env.params.query["action_takeout"]?.try &.to_i? action_takeout ||= 0 action_takeout = action_takeout == 1 @@ -104,33 +83,8 @@ module Invidious::Routes::Subscriptions if format == "json" env.response.content_type = "application/json" env.response.headers["content-disposition"] = "attachment" - playlists = Invidious::Database::Playlists.select_like_iv(user.email) - return JSON.build do |json| - json.object do - json.field "subscriptions", user.subscriptions - json.field "watch_history", user.watched - json.field "preferences", user.preferences - json.field "playlists" do - json.array do - playlists.each do |playlist| - json.object do - json.field "title", playlist.title - json.field "description", html_to_content(playlist.description_html) - json.field "privacy", playlist.privacy.to_s - json.field "videos" do - json.array do - Invidious::Database::PlaylistVideos.select_ids(playlist.id, playlist.index, limit: 500).each do |video_id| - json.string video_id - end - end - end - end - end - end - end - end - end + return Invidious::User::Export.to_invidious(user) else env.response.content_type = "application/xml" env.response.headers["content-disposition"] = "attachment" diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 3a92ef96..083087a9 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -21,7 +21,7 @@ module Invidious::Routes::VideoPlayback end # Sanity check, to avoid being used as an open proxy - if !host.matches?(/[\w-]+.googlevideo.com/) + if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/) return error_template(400, "Invalid \"host\" parameter.") end @@ -35,7 +35,15 @@ module Invidious::Routes::VideoPlayback end end - client = make_client(URI.parse(host), region) + # See: https://github.com/iv-org/invidious/issues/3302 + range_header = env.request.headers["Range"]? + sq = query_params["sq"]? + if range_header.nil? && sq.nil? + range_for_head = query_params["range"]? || "0-640" + headers["Range"] = "bytes=#{range_for_head}" + end + + client = make_client(URI.parse(host), region, force_resolve: true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -50,7 +58,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region) + client = make_client(URI.parse(new_host), region, force_resolve: true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -64,15 +72,23 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve: true) rescue ex error = ex.message end end + # Remove the Range header added previously. + headers.delete("Range") if range_header.nil? + + playback_statistics = get_playback_statistic() + playback_statistics["totalRequests"] += 1 + if response.status_code >= 400 env.response.content_type = "text/plain" haltf env, response.status_code + else + playback_statistics["successfulRequests"] += 1 end if url.includes? "&file=seg.ts" @@ -91,14 +107,8 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? - location = URI.parse(location) - location = "#{location.request_target}&host=#{location.host}" - - if region - location += "®ion=#{region}" - end - - return env.redirect location + url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) + return env.redirect url end IO.copy(resp.body_io, env.response) @@ -122,7 +132,7 @@ module Invidious::Routes::VideoPlayback end # TODO: Record bytes written so we can restart after a chunk fails - while true + loop do if !range_end && content_length range_end = content_length end @@ -155,10 +165,13 @@ module Invidious::Routes::VideoPlayback env.response.headers["Access-Control-Allow-Origin"] = "*" if location = resp.headers["Location"]? - location = URI.parse(location) - location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" + url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region) - env.redirect location + if title = query_params["title"]? + url = "#{url}&title=#{URI.encode_www_form(title)}" + end + + env.redirect url break end @@ -187,7 +200,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region) + client = make_client(URI.parse(host), region, force_resolve: true) end end @@ -244,6 +257,11 @@ module Invidious::Routes::VideoPlayback # YouTube /videoplayback links expire after 6 hours, # so we have a mechanism here to redirect to the latest version def self.latest_version(env) + if CONFIG.invidious_companion.present? + invidious_companion = CONFIG.invidious_companion.sample + return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}" + end + id = env.params.query["id"]? itag = env.params.query["itag"]?.try &.to_i? @@ -252,7 +270,7 @@ module Invidious::Routes::VideoPlayback return error_template(400, "Invalid video ID") end - if itag.nil? || itag <= 0 || itag >= 1000 + if !itag.nil? && (itag <= 0 || itag >= 1000) return error_template(400, "Invalid itag") end @@ -265,9 +283,19 @@ module Invidious::Routes::VideoPlayback return error_template(403, "Administrator has disabled this endpoint.") end - video = get_video(id, region: region) + begin + video = get_video(id, region: region) + rescue ex : NotFoundException + return error_template(404, ex) + rescue ex + return error_template(500, ex) + end - fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } + if itag.nil? + fmt = video.fmt_stream[-1]? + else + fmt = video.fmt_stream.find(nil) { |f| f["itag"].as_i == itag } || video.adaptive_fmts.find(nil) { |f| f["itag"].as_i == itag } + end url = fmt.try &.["url"]?.try &.as_s if !url diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 867ffa6a..e777b3f1 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -30,14 +30,6 @@ module Invidious::Routes::Watch return env.redirect "/" end - embed_link = "/embed/#{id}" - if env.params.query.size > 1 - embed_params = HTTP::Params.parse(env.params.query.to_s) - embed_params.delete_all("v") - embed_link += "?" - embed_link += embed_params.to_s - end - plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") continuation = process_continuation(env.params.query, plid, id) @@ -61,8 +53,9 @@ module Invidious::Routes::Watch begin video = get_video(id, region: params.region) - rescue ex : VideoRedirect - return env.redirect env.request.resource.gsub(id, ex.video_id) + rescue ex : NotFoundException + LOGGER.error("get_video not found: #{id} : #{ex.message}") + return error_template(404, ex) rescue ex LOGGER.error("get_video: #{id} : #{ex.message}") return error_template(500, ex) @@ -75,11 +68,11 @@ module Invidious::Routes::Watch end env.params.query.delete_all("iv_load_policy") - if watched && preferences.watch_history && !watched.includes? id + if watched && preferences.watch_history Invidious::Database::Users.mark_watched(user.as(User), id) end - if notifications && notifications.includes? id + if CONFIG.enable_user_notifications && notifications && notifications.includes? id Invidious::Database::Users.remove_notification(user.as(User), id) env.get("user").as(User).notifications.delete(id) notifications.delete(id) @@ -94,31 +87,31 @@ module Invidious::Routes::Watch if source == "youtube" begin - comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] rescue ex if preferences.comments[1] == "reddit" - comments, reddit_thread = fetch_reddit_comments(id) - comment_html = template_reddit_comments(comments, locale) + comments, reddit_thread = Comments.fetch_reddit(id) + comment_html = Frontend::Comments.template_reddit(comments, locale) - comment_html = fill_links(comment_html, "https", "www.reddit.com") - comment_html = replace_links(comment_html) + comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com") + comment_html = Comments.replace_links(comment_html) end end elsif source == "reddit" begin - comments, reddit_thread = fetch_reddit_comments(id) - comment_html = template_reddit_comments(comments, locale) + comments, reddit_thread = Comments.fetch_reddit(id) + comment_html = Frontend::Comments.template_reddit(comments, locale) - comment_html = fill_links(comment_html, "https", "www.reddit.com") - comment_html = replace_links(comment_html) + comment_html = Comments.fill_links(comment_html, "https", "www.reddit.com") + comment_html = Comments.replace_links(comment_html) rescue ex if preferences.comments[1] == "youtube" - comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] end end end else - comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] + comment_html = JSON.parse(Comments.fetch_youtube(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] end comment_html ||= "" @@ -128,10 +121,12 @@ module Invidious::Routes::Watch adaptive_fmts = video.adaptive_fmts if params.local - fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } - adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) } + fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } end + # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback + adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) } + video_streams = video.video_streams audio_streams = video.audio_streams @@ -197,6 +192,14 @@ module Invidious::Routes::Watch captions: video.captions ) + if CONFIG.invidious_companion.present? + invidious_companion = CONFIG.invidious_companion.sample + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src #{invidious_companion.public_url}") + .gsub("connect-src", "connect-src #{invidious_companion.public_url}") + end + templated "watch" end @@ -248,20 +251,10 @@ module Invidious::Routes::Watch end end - if env.params.query["action_mark_watched"]? - action = "action_mark_watched" - elsif env.params.query["action_mark_unwatched"]? - action = "action_mark_unwatched" - else - return env.redirect referer - end - - case action - when "action_mark_watched" - if !user.watched.includes? id - Invidious::Database::Users.mark_watched(user, id) - end - when "action_mark_unwatched" + case action = env.params.query["action"]? + when "mark_watched" + Invidious::Database::Users.mark_watched(user, id) + when "mark_unwatched" Invidious::Database::Users.mark_unwatched(user, id) else return error_json(400, "Unsupported action #{action}") @@ -284,6 +277,12 @@ module Invidious::Routes::Watch return error_template(400, "Invalid clip ID") if response["error"]? if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") + if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s + start_time, end_time, _ = parse_clip_parameters(params) + env.params.query["start"] = start_time.to_s if start_time != nil + env.params.query["end"] = end_time.to_s if end_time != nil + end + return env.redirect "/watch?v=#{video_id}&#{env.params.query}" else return error_template(404, "The requested clip doesn't exist") @@ -294,6 +293,9 @@ module Invidious::Routes::Watch if CONFIG.disabled?("downloads") return error_template(403, "Administrator has disabled this endpoint.") end + if CONFIG.invidious_companion.present? + return error_template(403, "Downloads should be routed through Companion when present") + end title = env.params.body["title"]? || "" video_id = env.params.body["id"]? || "" @@ -306,27 +308,27 @@ module Invidious::Routes::Watch download_widget = JSON.parse(selection) extension = download_widget["ext"].as_s - filename = "#{video_id}-#{title}.#{extension}" + filename = "#{title}-#{video_id}.#{extension}" - # Pass form parameters as URL parameters for the handlers of both - # /latest_version and /api/v1/captions. This avoids an un-necessary - # redirect and duplicated (and hazardous) sanity checks. - env.params.query["id"] = video_id - env.params.query["title"] = filename - - # Delete the useless ones + # Delete the now useless URL parameters env.params.body.delete("id") env.params.body.delete("title") env.params.body.delete("download_widget") + # Pass form parameters as URL parameters for the handlers of both + # /latest_version and /api/v1/captions. This avoids an un-necessary + # redirect and duplicated (and hazardous) sanity checks. if label = download_widget["label"]? # URL params specific to /api/v1/captions/:id - env.params.query["label"] = URI.encode_www_form(label.as_s, space_to_plus: false) + env.params.url["id"] = video_id + env.params.query["title"] = filename + env.params.query["label"] = URI.decode_www_form(label.as_s) return Invidious::Routes::API::V1::Videos.captions(env) - elsif itag = download_widget["itag"]?.try &.as_i + elsif itag = download_widget["itag"]?.try &.as_i.to_s # URL params specific to /latest_version - env.params.query["itag"] = itag.to_s + env.params.query["id"] = video_id + env.params.query["title"] = filename env.params.query["local"] = "true" return Invidious::Routes::VideoPlayback.latest_version(env) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index bd72c577..46b71f1f 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -1,130 +1,319 @@ module Invidious::Routing - {% for http_method in {"get", "post", "delete", "options", "patch", "put", "head"} %} + extend self + + {% for http_method in {"get", "post", "delete", "options", "patch", "put"} %} macro {{http_method.id}}(path, controller, method = :handle) - {{http_method.id}} \{{ path }} do |env| + unless Kemal::Utils.path_starts_with_slash?(\{{path}}) + raise Kemal::Exceptions::InvalidPathStartException.new({{http_method}}, \{{path}}) + end + + Kemal::RouteHandler::INSTANCE.add_route({{http_method.upcase}}, \{{path}}) do |env| \{{ controller }}.\{{ method.id }}(env) end end {% end %} -end - -macro define_user_routes - # User login/out - Invidious::Routing.get "/login", Invidious::Routes::Login, :login_page - Invidious::Routing.post "/login", Invidious::Routes::Login, :login - Invidious::Routing.post "/signout", Invidious::Routes::Login, :signout - Invidious::Routing.get "/Captcha", Invidious::Routes::Login, :captcha - - # User preferences - Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :show - Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update - Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme - Invidious::Routing.get "/data_control", Invidious::Routes::PreferencesRoute, :data_control - Invidious::Routing.post "/data_control", Invidious::Routes::PreferencesRoute, :update_data_control - - # User account management - Invidious::Routing.get "/change_password", Invidious::Routes::Account, :get_change_password - Invidious::Routing.post "/change_password", Invidious::Routes::Account, :post_change_password - Invidious::Routing.get "/delete_account", Invidious::Routes::Account, :get_delete - Invidious::Routing.post "/delete_account", Invidious::Routes::Account, :post_delete - Invidious::Routing.get "/clear_watch_history", Invidious::Routes::Account, :get_clear_history - Invidious::Routing.post "/clear_watch_history", Invidious::Routes::Account, :post_clear_history - Invidious::Routing.get "/authorize_token", Invidious::Routes::Account, :get_authorize_token - Invidious::Routing.post "/authorize_token", Invidious::Routes::Account, :post_authorize_token - Invidious::Routing.get "/token_manager", Invidious::Routes::Account, :token_manager - Invidious::Routing.post "/token_ajax", Invidious::Routes::Account, :token_ajax -end - -macro define_v1_api_routes - {{namespace = Invidious::Routes::API::V1}} - # Videos - Invidious::Routing.get "/api/v1/videos/:id", {{namespace}}::Videos, :videos - Invidious::Routing.get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards - Invidious::Routing.get "/api/v1/captions/:id", {{namespace}}::Videos, :captions - Invidious::Routing.get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations - Invidious::Routing.get "/api/v1/comments/:id", {{namespace}}::Videos, :comments - - # Feeds - Invidious::Routing.get "/api/v1/trending", {{namespace}}::Feeds, :trending - Invidious::Routing.get "/api/v1/popular", {{namespace}}::Feeds, :popular - - # Channels - Invidious::Routing.get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home - {% for route in {"videos", "latest", "playlists", "community", "search"} %} - Invidious::Routing.get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} - Invidious::Routing.get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} - {% end %} - - # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community - Invidious::Routing.get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect - Invidious::Routing.get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect - - - # Search - Invidious::Routing.get "/api/v1/search", {{namespace}}::Search, :search - Invidious::Routing.get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions - - # Authenticated - - # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr - # - # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - - Invidious::Routing.get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences - Invidious::Routing.post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences - - Invidious::Routing.get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed - - Invidious::Routing.get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions - Invidious::Routing.post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel - Invidious::Routing.delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel - - - Invidious::Routing.get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists - Invidious::Routing.post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist - Invidious::Routing.patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute - Invidious::Routing.delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist - - - Invidious::Routing.post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist - Invidious::Routing.delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist - - Invidious::Routing.get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens - Invidious::Routing.post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token - Invidious::Routing.post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token - - Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - - # Misc - Invidious::Routing.get "/api/v1/stats", {{namespace}}::Misc, :stats - Invidious::Routing.get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist - Invidious::Routing.get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist - Invidious::Routing.get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes -end - -macro define_api_manifest_routes - Invidious::Routing.get "/api/manifest/dash/id/:id", Invidious::Routes::API::Manifest, :get_dash_video_id - - Invidious::Routing.get "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :get_dash_video_playback - Invidious::Routing.get "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :get_dash_video_playback_greedy - - Invidious::Routing.options "/api/manifest/dash/id/videoplayback", Invidious::Routes::API::Manifest, :options_dash_video_playback - Invidious::Routing.options "/api/manifest/dash/id/videoplayback/*", Invidious::Routes::API::Manifest, :options_dash_video_playback - - Invidious::Routing.get "/api/manifest/hls_playlist/*", Invidious::Routes::API::Manifest, :get_hls_playlist - Invidious::Routing.get "/api/manifest/hls_variant/*", Invidious::Routes::API::Manifest, :get_hls_variant -end - -macro define_video_playback_routes - Invidious::Routing.get "/videoplayback", Invidious::Routes::VideoPlayback, :get_video_playback - Invidious::Routing.get "/videoplayback/*", Invidious::Routes::VideoPlayback, :get_video_playback_greedy - - Invidious::Routing.options "/videoplayback", Invidious::Routes::VideoPlayback, :options_video_playback - Invidious::Routing.options "/videoplayback/*", Invidious::Routes::VideoPlayback, :options_video_playback - - Invidious::Routing.get "/latest_version", Invidious::Routes::VideoPlayback, :latest_version + + def register_all + {% unless flag?(:api_only) %} + get "/", Routes::Misc, :home + get "/privacy", Routes::Misc, :privacy + get "/licenses", Routes::Misc, :licenses + get "/redirect", Routes::Misc, :cross_instance_redirect + + self.register_channel_routes + self.register_watch_routes + + self.register_iv_playlist_routes + self.register_yt_playlist_routes + + self.register_search_routes + + self.register_user_routes + self.register_feed_routes + + # Support push notifications via PubSubHubbub + get "/feed/webhook/:token", Routes::Feeds, :push_notifications_get + post "/feed/webhook/:token", Routes::Feeds, :push_notifications_post + + if CONFIG.enable_user_notifications + get "/modify_notifications", Routes::Notifications, :modify + end + {% end %} + + self.register_image_routes + self.register_api_v1_routes + self.register_api_manifest_routes + self.register_video_playback_routes + end + + # ------------------- + # Invidious routes + # ------------------- + + def register_user_routes + # User login/out + get "/login", Routes::Login, :login_page + post "/login", Routes::Login, :login + post "/signout", Routes::Login, :signout + + # User preferences + get "/preferences", Routes::PreferencesRoute, :show + post "/preferences", Routes::PreferencesRoute, :update + get "/toggle_theme", Routes::PreferencesRoute, :toggle_theme + get "/data_control", Routes::PreferencesRoute, :data_control + post "/data_control", Routes::PreferencesRoute, :update_data_control + + # User account management + get "/change_password", Routes::Account, :get_change_password + post "/change_password", Routes::Account, :post_change_password + get "/delete_account", Routes::Account, :get_delete + post "/delete_account", Routes::Account, :post_delete + get "/clear_watch_history", Routes::Account, :get_clear_history + post "/clear_watch_history", Routes::Account, :post_clear_history + get "/authorize_token", Routes::Account, :get_authorize_token + post "/authorize_token", Routes::Account, :post_authorize_token + get "/token_manager", Routes::Account, :token_manager + post "/token_ajax", Routes::Account, :token_ajax + post "/subscription_ajax", Routes::Subscriptions, :toggle_subscription + get "/subscription_manager", Routes::Subscriptions, :subscription_manager + end + + def register_iv_playlist_routes + get "/create_playlist", Routes::Playlists, :new + post "/create_playlist", Routes::Playlists, :create + get "/subscribe_playlist", Routes::Playlists, :subscribe + get "/delete_playlist", Routes::Playlists, :delete_page + post "/delete_playlist", Routes::Playlists, :delete + get "/edit_playlist", Routes::Playlists, :edit + post "/edit_playlist", Routes::Playlists, :update + get "/add_playlist_items", Routes::Playlists, :add_playlist_items_page + post "/playlist_ajax", Routes::Playlists, :playlist_ajax + end + + def register_feed_routes + # Feeds + get "/view_all_playlists", Routes::Feeds, :view_all_playlists_redirect + get "/feed/playlists", Routes::Feeds, :playlists + get "/feed/popular", Routes::Feeds, :popular + get "/feed/trending", Routes::Feeds, :trending + get "/feed/subscriptions", Routes::Feeds, :subscriptions + get "/feed/history", Routes::Feeds, :history + + # RSS Feeds + get "/feed/channel/:ucid", Routes::Feeds, :rss_channel + get "/feed/private", Routes::Feeds, :rss_private + get "/feed/playlist/:plid", Routes::Feeds, :rss_playlist + get "/feeds/videos.xml", Routes::Feeds, :rss_videos + end + + # ------------------- + # Youtube routes + # ------------------- + + def register_channel_routes + get "/channel/:ucid", Routes::Channels, :home + get "/channel/:ucid/home", Routes::Channels, :home + get "/channel/:ucid/videos", Routes::Channels, :videos + get "/channel/:ucid/shorts", Routes::Channels, :shorts + get "/channel/:ucid/streams", Routes::Channels, :streams + get "/channel/:ucid/podcasts", Routes::Channels, :podcasts + get "/channel/:ucid/releases", Routes::Channels, :releases + get "/channel/:ucid/courses", Routes::Channels, :courses + get "/channel/:ucid/playlists", Routes::Channels, :playlists + get "/channel/:ucid/community", Routes::Channels, :community + get "/channel/:ucid/posts", Routes::Channels, :community + get "/channel/:ucid/channels", Routes::Channels, :channels + get "/channel/:ucid/about", Routes::Channels, :about + + get "/channel/:ucid/live", Routes::Channels, :live + get "/user/:user/live", Routes::Channels, :live + get "/c/:user/live", Routes::Channels, :live + get "/post/:id", Routes::Channels, :post + + # Channel catch-all, to redirect future routes to the channel's home + # NOTE: defined last in order to be processed after the other routes + get "/channel/:ucid/*", Routes::Channels, :redirect_home + + # /c/LinusTechTips + get "/c/:user", Routes::Channels, :brand_redirect + get "/c/:user/:tab", Routes::Channels, :brand_redirect + + # /user/linustechtips (Not always the same as /c/) + get "/user/:user", Routes::Channels, :brand_redirect + get "/user/:user/:tab", Routes::Channels, :brand_redirect + + # /@LinusTechTips (Handle) + get "/@:user", Routes::Channels, :brand_redirect + get "/@:user/:tab", Routes::Channels, :brand_redirect + + # /attribution_link?a=anything&u=/channel/UCZYTClx2T1of7BRZ86-8fow + get "/attribution_link", Routes::Channels, :brand_redirect + get "/attribution_link/:tab", Routes::Channels, :brand_redirect + + # /profile?user=linustechtips + get "/profile", Routes::Channels, :profile + get "/profile/*", Routes::Channels, :profile + end + + def register_watch_routes + get "/watch", Routes::Watch, :handle + post "/watch_ajax", Routes::Watch, :mark_watched + get "/watch/:id", Routes::Watch, :redirect + get "/live/:id", Routes::Watch, :redirect + get "/shorts/:id", Routes::Watch, :redirect + get "/clip/:clip", Routes::Watch, :clip + get "/w/:id", Routes::Watch, :redirect + get "/v/:id", Routes::Watch, :redirect + get "/e/:id", Routes::Watch, :redirect + + post "/download", Routes::Watch, :download + + get "/embed/", Routes::Embed, :redirect + get "/embed/:id", Routes::Embed, :show + end + + def register_yt_playlist_routes + get "/playlist", Routes::Playlists, :show + get "/mix", Routes::Playlists, :mix + get "/watch_videos", Routes::Playlists, :watch_videos + end + + def register_search_routes + get "/opensearch.xml", Routes::Search, :opensearch + get "/results", Routes::Search, :results + get "/search", Routes::Search, :search + get "/hashtag/:hashtag", Routes::Search, :hashtag + end + + # ------------------- + # Media proxy routes + # ------------------- + + def register_api_manifest_routes + get "/api/manifest/dash/id/:id", Routes::API::Manifest, :get_dash_video_id + + get "/api/manifest/dash/id/videoplayback", Routes::API::Manifest, :get_dash_video_playback + get "/api/manifest/dash/id/videoplayback/*", Routes::API::Manifest, :get_dash_video_playback_greedy + + options "/api/manifest/dash/id/videoplayback", Routes::API::Manifest, :options_dash_video_playback + options "/api/manifest/dash/id/videoplayback/*", Routes::API::Manifest, :options_dash_video_playback + + get "/api/manifest/hls_playlist/*", Routes::API::Manifest, :get_hls_playlist + get "/api/manifest/hls_variant/*", Routes::API::Manifest, :get_hls_variant + end + + def register_video_playback_routes + get "/videoplayback", Routes::VideoPlayback, :get_video_playback + get "/videoplayback/*", Routes::VideoPlayback, :get_video_playback_greedy + + options "/videoplayback", Routes::VideoPlayback, :options_video_playback + options "/videoplayback/*", Routes::VideoPlayback, :options_video_playback + + get "/latest_version", Routes::VideoPlayback, :latest_version + end + + def register_image_routes + get "/ggpht/*", Routes::Images, :ggpht + options "/sb/:authority/:id/:storyboard/:index", Routes::Images, :options_storyboard + get "/sb/:authority/:id/:storyboard/:index", Routes::Images, :get_storyboard + get "/s_p/:id/:name", Routes::Images, :s_p_image + get "/yts/img/:name", Routes::Images, :yts_image + get "/vi/:id/:name", Routes::Images, :thumbnails + end + + # ------------------- + # API routes + # ------------------- + + def register_api_v1_routes + {% begin %} + {{namespace = Routes::API::V1}} + + # Videos + get "/api/v1/videos/:id", {{namespace}}::Videos, :videos + get "/api/v1/storyboards/:id", {{namespace}}::Videos, :storyboards + get "/api/v1/captions/:id", {{namespace}}::Videos, :captions + get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations + get "/api/v1/comments/:id", {{namespace}}::Videos, :comments + get "/api/v1/clips/:id", {{namespace}}::Videos, :clips + get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts + + # Feeds + get "/api/v1/trending", {{namespace}}::Feeds, :trending + get "/api/v1/popular", {{namespace}}::Feeds, :popular + + # Channels + get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest + get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos + get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts + get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams + get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts + get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases + get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses + get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists + get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community + get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community + get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels + get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search + + # Posts + get "/api/v1/post/:id", {{namespace}}::Channels, :post + get "/api/v1/post/:id/comments", {{namespace}}::Channels, :post_comments + + # 301 redirects to new /api/v1/channels/community/:ucid and /:ucid/community + get "/api/v1/channels/comments/:ucid", {{namespace}}::Channels, :channel_comments_redirect + get "/api/v1/channels/:ucid/comments", {{namespace}}::Channels, :channel_comments_redirect + + # Search + get "/api/v1/search", {{namespace}}::Search, :search + get "/api/v1/search/suggestions", {{namespace}}::Search, :search_suggestions + get "/api/v1/hashtag/:hashtag", {{namespace}}::Search, :hashtag + + + # Authenticated + + get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences + post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences + + get "/api/v1/auth/export/invidious", {{namespace}}::Authenticated, :export_invidious + post "/api/v1/auth/import/invidious", {{namespace}}::Authenticated, :import_invidious + + get "/api/v1/auth/history", {{namespace}}::Authenticated, :get_history + post "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_watched + delete "/api/v1/auth/history/:id", {{namespace}}::Authenticated, :mark_unwatched + delete "/api/v1/auth/history", {{namespace}}::Authenticated, :clear_history + + get "/api/v1/auth/feed", {{namespace}}::Authenticated, :feed + + get "/api/v1/auth/subscriptions", {{namespace}}::Authenticated, :get_subscriptions + post "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :subscribe_channel + delete "/api/v1/auth/subscriptions/:ucid", {{namespace}}::Authenticated, :unsubscribe_channel + + get "/api/v1/auth/playlists", {{namespace}}::Authenticated, :list_playlists + post "/api/v1/auth/playlists", {{namespace}}::Authenticated, :create_playlist + patch "/api/v1/auth/playlists/:plid",{{namespace}}:: Authenticated, :update_playlist_attribute + delete "/api/v1/auth/playlists/:plid", {{namespace}}::Authenticated, :delete_playlist + post "/api/v1/auth/playlists/:plid/videos", {{namespace}}::Authenticated, :insert_video_into_playlist + delete "/api/v1/auth/playlists/:plid/videos/:index", {{namespace}}::Authenticated, :delete_video_in_playlist + + get "/api/v1/auth/tokens", {{namespace}}::Authenticated, :get_tokens + post "/api/v1/auth/tokens/register", {{namespace}}::Authenticated, :register_token + post "/api/v1/auth/tokens/unregister", {{namespace}}::Authenticated, :unregister_token + + if CONFIG.enable_user_notifications + get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications + end + + # Misc + get "/api/v1/stats", {{namespace}}::Misc, :stats + get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist + get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist + get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes + get "/api/v1/resolveurl", {{namespace}}::Misc, :resolve_url + {% end %} + end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr deleted file mode 100644 index ae106bf6..00000000 --- a/src/invidious/search.cr +++ /dev/null @@ -1,254 +0,0 @@ -class ChannelSearchException < InfoException - getter channel : String - - def initialize(@channel) - end -end - -def channel_search(query, page, channel) : Array(SearchItem) - response = YT_POOL.client &.get("/channel/#{channel}") - - if response.status_code == 404 - response = YT_POOL.client &.get("/user/#{channel}") - response = YT_POOL.client &.get("/c/#{channel}") if response.status_code == 404 - initial_data = extract_initial_data(response.body) - ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) - raise ChannelSearchException.new(channel) if !ucid - else - ucid = channel - end - - continuation = produce_channel_search_continuation(ucid, query, page) - response_json = YoutubeAPI.browse(continuation) - - continuation_items = response_json["onResponseReceivedActions"]? - .try &.[0]["appendContinuationItemsAction"]["continuationItems"] - - return [] of SearchItem if !continuation_items - - items = [] of SearchItem - continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item| - extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t } - end - - return items -end - -def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem) - return [] of SearchItem if query.empty? - - client_config = YoutubeAPI::ClientConfig.new(region: region) - initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) - - return extract_items(initial_data) -end - -def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "", - duration : String = "", features : Array(String) = [] of String) - object = { - "1:varint" => 0_i64, - "2:embedded" => {} of String => Int64, - "9:varint" => ((page - 1) * 20).to_i64, - } - - case sort - when "relevance" - object["1:varint"] = 0_i64 - when "rating" - object["1:varint"] = 1_i64 - when "upload_date", "date" - object["1:varint"] = 2_i64 - when "view_count", "views" - object["1:varint"] = 3_i64 - else - raise "No sort #{sort}" - end - - case date - when "hour" - object["2:embedded"].as(Hash)["1:varint"] = 1_i64 - when "today" - object["2:embedded"].as(Hash)["1:varint"] = 2_i64 - when "week" - object["2:embedded"].as(Hash)["1:varint"] = 3_i64 - when "month" - object["2:embedded"].as(Hash)["1:varint"] = 4_i64 - when "year" - object["2:embedded"].as(Hash)["1:varint"] = 5_i64 - else nil # Ignore - end - - case content_type - when "video" - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - when "channel" - object["2:embedded"].as(Hash)["2:varint"] = 2_i64 - when "playlist" - object["2:embedded"].as(Hash)["2:varint"] = 3_i64 - when "movie" - object["2:embedded"].as(Hash)["2:varint"] = 4_i64 - when "show" - object["2:embedded"].as(Hash)["2:varint"] = 5_i64 - when "all" - # - else - object["2:embedded"].as(Hash)["2:varint"] = 1_i64 - end - - case duration - when "short" - object["2:embedded"].as(Hash)["3:varint"] = 1_i64 - when "long" - object["2:embedded"].as(Hash)["3:varint"] = 2_i64 - else nil # Ignore - end - - features.each do |feature| - case feature - when "hd" - object["2:embedded"].as(Hash)["4:varint"] = 1_i64 - when "subtitles" - object["2:embedded"].as(Hash)["5:varint"] = 1_i64 - when "creative_commons", "cc" - object["2:embedded"].as(Hash)["6:varint"] = 1_i64 - when "3d" - object["2:embedded"].as(Hash)["7:varint"] = 1_i64 - when "live", "livestream" - object["2:embedded"].as(Hash)["8:varint"] = 1_i64 - when "purchased" - object["2:embedded"].as(Hash)["9:varint"] = 1_i64 - when "4k" - object["2:embedded"].as(Hash)["14:varint"] = 1_i64 - when "360" - object["2:embedded"].as(Hash)["15:varint"] = 1_i64 - when "location" - object["2:embedded"].as(Hash)["23:varint"] = 1_i64 - when "hdr" - object["2:embedded"].as(Hash)["25:varint"] = 1_i64 - else nil # Ignore - end - end - - if object["2:embedded"].as(Hash).empty? - object.delete("2:embedded") - end - - params = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return params -end - -def produce_channel_search_continuation(ucid, query, page) - if page <= 1 - idx = 0_i64 - else - idx = 30_i64 * (page - 1) - end - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:base64" => { - "2:string" => "search", - "6:varint" => 1_i64, - "7:varint" => 1_i64, - "12:varint" => 1_i64, - "15:base64" => { - "3:varint" => idx, - }, - "23:varint" => 0_i64, - }, - "11:string" => query, - "35:string" => "browse-feed#{ucid}search", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def process_search_query(query, page, user, region) - if user - user = user.as(Invidious::User) - view_name = "subscriptions_#{sha256(user.email)}" - end - - channel = nil - content_type = "all" - date = "" - duration = "" - features = [] of String - sort = "relevance" - subscriptions = nil - - operators = query.split(" ").select(&.match(/\w+:[\w,]+/)) - operators.each do |operator| - key, value = operator.downcase.split(":") - - case key - when "channel", "user" - channel = operator.split(":")[-1] - when "content_type", "type" - content_type = value - when "date" - date = value - when "duration" - duration = value - when "feature", "features" - features = value.split(",") - when "sort" - sort = value - when "subscriptions" - subscriptions = value == "true" - else - operators.delete(operator) - end - end - - search_query = (query.split(" ") - operators).join(" ") - - if channel - items = channel_search(search_query, page, channel) - elsif subscriptions - if view_name - items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM ( - SELECT *, - to_tsvector(#{view_name}.title) || - to_tsvector(#{view_name}.author) - as document - FROM #{view_name} - ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) - else - items = [] of ChannelVideo - end - else - search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type, - duration: duration, features: features) - - items = search(search_query, search_params, region) - end - - # Light processing to flatten search results out of Categories. - # They should ideally be supported in the future. - items_without_category = [] of SearchItem | ChannelVideo - items.each do |i| - if i.is_a? Category - i.contents.each do |nest_i| - if !nest_i.is_a? Video - items_without_category << nest_i - end - end - else - items_without_category << i - end - end - - {search_query, items_without_category, operators} -end diff --git a/src/invidious/search/ctoken.cr b/src/invidious/search/ctoken.cr new file mode 100644 index 00000000..161065e0 --- /dev/null +++ b/src/invidious/search/ctoken.cr @@ -0,0 +1,32 @@ +def produce_channel_search_continuation(ucid, query, page) + if page <= 1 + idx = 0_i64 + else + idx = 30_i64 * (page - 1) + end + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:base64" => { + "2:string" => "search", + "6:varint" => 1_i64, + "7:varint" => 1_i64, + "12:varint" => 1_i64, + "15:base64" => { + "3:varint" => idx, + }, + "23:varint" => 0_i64, + }, + "11:string" => query, + "35:string" => "browse-feed#{ucid}search", + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr new file mode 100644 index 00000000..bc2715cf --- /dev/null +++ b/src/invidious/search/filters.cr @@ -0,0 +1,376 @@ +require "protodec/utils" +require "http/params" + +module Invidious::Search + struct Filters + # Values correspond to { "2:embedded": { "1:varint":
- <% videos.each_slice(4) do |slice| %>
- <% slice.each do |item| %>
- <%= rendered "components/item" %>
- <% end %>
- <% end %>
-
-<% if query %>
-
-
-<% end %>
+<%= rendered "components/items_paginated" %>
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 40b553a9..686de6bd 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -1,116 +1,59 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url =
+ case selected_tab
+ when .shorts? then "/channel/#{ucid}/shorts"
+ when .streams? then "/channel/#{ucid}/streams"
+ when .playlists? then "/channel/#{ucid}/playlists"
+ when .channels? then "/channel/#{ucid}/channels"
+ when .podcasts? then "/channel/#{ucid}/podcasts"
+ when .releases? then "/channel/#{ucid}/releases"
+ when .courses? then "/channel/#{ucid}/courses"
+ else
+ "/channel/#{ucid}"
+ end
+
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
+ base_url: relative_url,
+ ctoken: next_continuation,
+ first_page: continuation.nil?,
+ params: env.params.query,
+ )
+%>
<% content_for "header" do %>
-
- <% if page > 1 %>
-
- <%= translate(locale, "Previous page") %>
-
- <% end %>
-
-
-
- <% if videos.size >= 20 %>
-
- <%= translate(locale, "Next page") %>
-
- <% end %>
-
-
- ">
-
-
- ">
-
-
-
-<% end %>
-
-
-
--
.request_target %>)
-
-
-
-
-
-<%= channel.description_html %>
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
+<%= rendered "components/channel_info" %>
- <%= translate(locale, "View channel on YouTube") %>
-
-
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
- <% if !channel.auto_generated %>
-
- <%= translate(locale, "Videos") %>
-
- <% end %>
-
- <% if channel.auto_generated %>
- <%= translate(locale, "Playlists") %>
- <% else %>
- <%= translate(locale, "Playlists") %>
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
-
-
-
- <% sort_options.each do |sort| %>
-
-
- <% if sort_by == sort %>
- <%= translate(locale, sort) %>
- <% else %>
-
- <%= translate(locale, sort) %>
-
- <% end %>
-
- <% end %>
-
-<% items.each do |item| %>
- <%= rendered "components/item" %>
-<% end %>
-
-
-
+<%= rendered "components/items_paginated" %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index f0add06b..132e636c 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -1,71 +1,21 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
+<%-
+ ucid = channel.ucid
+ author = HTML.escape(channel.author)
+ channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+ relative_url = "/channel/#{ucid}/community"
+ youtube_url = "https://www.youtube.com#{relative_url}"
+ redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+ selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Posts
+-%>
<% content_for "header" do %>
+
- <% if page > 1 %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Previous page") %>
-
- <% end %>
-
-
-
- <% if count == 60 %>
- &sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
- <%= translate(locale, "Next page") %>
-
- <% end %>
-
-
- ">
-
-
- ">
-
-
-
-<% end %>
-
-
-
--
-
-
-
-
-<%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %>
-
- <% sub_count_text = number_to_short_text(channel.sub_count) %>
- <%= rendered "components/subscribe_widget" %>
-
-
-
-
+<%= rendered "components/channel_info" %>
- <%= translate(locale, "View channel on YouTube") %>
-
-
-
- <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
- "><%= translate(locale, "Switch Invidious Instance") %>
- <% else %>
- <%= translate(locale, "Switch Invidious Instance") %>
- <% end %>
-
- <% if !channel.auto_generated %>
-
- <% end %>
-
-
- <% if channel.tabs.includes? "community" %>
- <%= translate(locale, "Community") %>
- <% end %>
-
- @@ -76,8 +26,8 @@
<%= error_message %>
- <%= template_youtube_comments(items.not_nil!, locale, thin_mode) %>
+
+ <%= IV::Frontend::Comments.template_youtube(items.not_nil!, locale, thin_mode) %>
<% end %>
diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr
new file mode 100644
index 00000000..f4164f31
--- /dev/null
+++ b/src/invidious/views/components/channel_info.ecr
@@ -0,0 +1,61 @@
+<% if channel.banner %>
+
+ " alt="" />
+
+
+ " alt="" />
+
+
+
+<% end %>
+
++
+
+
+
+
+
+
+
+
+ <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %>
+
+
+ <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %>
+
+
+
+
+<%= channel.description_html %>
+
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr
index 5f8bde13..a24423df 100644
--- a/src/invidious/views/components/item.ecr
+++ b/src/invidious/views/components/item.ecr
@@ -1,151 +1,199 @@
+<%-
+ thin_mode = env.get("preferences").as(Preferences).thin_mode
+ item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category | ProblematicTimelineItem) && env.get?("user").try &.as(User).watched.index(item.id) != nil
+ author_verified = item.responds_to?(:author_verified) && item.author_verified
+-%>
+
+
+
+
+ <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
+
+
+
+
+ <% sort_options.each do |sort| %>
+
+
+ <% if sort_by == sort %>
+ <%= translate(locale, sort) %>
+ <% else %>
+ <%= translate(locale, sort) %>
+ <% end %>
+
+ <% end %>
+
<% case item when %>
<% when SearchChannel %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
+ <% if !thin_mode %>
+
- "/>
+ " alt="" />
- <% end %>
-
"/>
+ " alt="" />
<%= HTML.escape(item.author) %>
- -<%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>
- <% if !item.auto_generated %><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
<% end %> -<%= item.description_html %>
- <% when SearchPlaylist, InvidiousPlaylist %> - <% if item.id.starts_with? "RD" %> - <% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %> - <% else %> - <% url = "/playlist?list=#{item.id}" %> + + <%- else -%> + <% end %> - - <% if !env.get("preferences").as(Preferences).thin_mode %> -
- "/>
-
- <% end %>
- <%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
-<%= HTML.escape(item.title) %>
- - -<%= HTML.escape(item.author) %>
- - <% when MixVideo %> - - <% if !env.get("preferences").as(Preferences).thin_mode %> -
-
- <% if item.length_seconds != 0 %>
-
- <% end %>
-
- <% if item.length_seconds != 0 %>
- <%= recode_length_seconds(item.length_seconds) %>
- <% end %> -<%= HTML.escape(item.title) %>
- - -<%= HTML.escape(item.author) %>
- - <% when PlaylistVideo %> - - <% if !env.get("preferences").as(Preferences).thin_mode %> -
-
- <% if plid = env.get?("remove_playlist_items") %>
-
- <% end %>
-
- <% if item.responds_to?(:live_now) && item.live_now %>
-
- <% end %>
-
- <% if plid = env.get?("remove_playlist_items") %>
-
- <% end %>
-
- <% if item.responds_to?(:live_now) && item.live_now %>
- <%= translate(locale, "LIVE") %>
- <% elsif item.length_seconds != 0 %> -<%= recode_length_seconds(item.length_seconds) %>
- <% end %> -<%= HTML.escape(item.title) %>
- -
- <% endpoint_params = "?v=#{item.id}&list=#{item.plid}" %>
- <%= rendered "components/video-context-buttons" %>
+
+
+ <% if !item.channel_handle.nil? %><%= item.channel_handle %>
<% end %> +<%= translate_count(locale, "generic_subscribers_count", item.subscriber_count, NumberFormatting::Separator) %>
+ <% if !item.auto_generated && item.channel_handle.nil? %><%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
<% end %> +<%= item.description_html %>
+ <% when SearchHashtag %> + <% if !thin_mode %> + +
+ <%- if item.video_count != 0 -%>
+
+
+ <%= translate_count(locale, "generic_videos_count", item.video_count, NumberFormatting::Separator) %>
+ <%- end -%> +
+ <%- if item.channel_count != 0 -%>
+
+ <% when SearchPlaylist, InvidiousPlaylist %>
+ <%-
+ if item.id.starts_with? "RD"
+ link_url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}"
+ else
+ link_url = "/playlist?list=#{item.id}"
+ end
+ -%>
+
+ <%= translate_count(locale, "generic_channels_count", item.channel_count, NumberFormatting::Separator) %>
+ <%- end -%> +
+ <%- if !thin_mode %>
+
+ " alt="" />
+
+ <%- else -%>
+
+ <%- end -%>
+
+
+
+
+
- <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
-
+ <%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
- <% elsif Time.utc - item.published > 1.minute %> -<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
+ <% if !item.ucid.to_s.empty? %> + +<%= HTML.escape(item.author) %> + <%- if author_verified %> <% end -%> +
+ + <% else %> +<%= HTML.escape(item.author) %> + <%- if author_verified %> <% end -%> +
+ <% end %> +
+
+ <% else %>
+ <%-
+ # `endpoint_params` is used for the "video-context-buttons" component
+ if item.is_a?(PlaylistVideo)
+ link_url = "/watch?v=#{item.id}&list=#{item.plid}&index=#{item.index}"
+ endpoint_params = "?v=#{item.id}&list=#{item.plid}"
+ elsif item.is_a?(MixVideo)
+ link_url = "/watch?v=#{item.id}&list=#{item.rdid}"
+ endpoint_params = "?v=#{item.id}&list=#{item.rdid}"
+ else
+ link_url = "/watch?v=#{item.id}"
+ endpoint_params = "?v=#{item.id}"
+ end
+ -%>
+
+
+
+
+ <%=translate(locale, "timeline_parse_error_placeholder_heading")%>
+<%=translate(locale, "timeline_parse_error_placeholder_message")%>
+
+
+ <%=translate(locale, "timeline_parse_error_show_technical_details")%>
+<%=get_issue_template(env, item.parse_exception)[1]%>+
+ <%- if !thin_mode -%>
+
+
+
+ <% if item_watched %>
+
+
+ <% end %>
+
+ <%- else -%>
+
+ <%- end -%>
+
+
+
+
+
+
+
+
+
+
+ <% if item_watched %>
+
+
+ <% end %>
+
+ <%- else -%>
+
+ <%- end -%>
+
+
+
+
+
+
- <% when Category %>
- <% else %>
-
- <% if !env.get("preferences").as(Preferences).thin_mode %>
-
+ <% if !item.ucid.to_s.empty? %>
+
+
- <% if item.responds_to?(:views) && item.views %>
- <%= HTML.escape(item.author) %> + <%- if author_verified %> <% end -%> +
+ + <% else %> +<%= HTML.escape(item.author) %> + <%- if author_verified %> <% end -%> +
<% end %>
-
- <% end %>
- <%= translate_count(locale, "generic_views_count", item.views || 0, NumberFormatting::Short) %>
-
-
- <% if env.get? "show_watched" %>
-
- <% elsif plid = env.get? "add_playlist_items" %>
-
- <% end %>
-
- <% if item.responds_to?(:live_now) && item.live_now %>
-
- <% end %>
-
- <% if env.get? "show_watched" %>
-
- <% elsif plid = env.get? "add_playlist_items" %>
-
- <% end %>
-
- <% if item.responds_to?(:live_now) && item.live_now %>
- <%= translate(locale, "LIVE") %>
- <% elsif item.length_seconds != 0 %> -<%= recode_length_seconds(item.length_seconds) %>
- <% end %> -<%= HTML.escape(item.title) %>
- - -
-
-
- <% endpoint_params = "?v=#{item.id}" %>
<%= rendered "components/video-context-buttons" %>
@@ -153,7 +201,7 @@
<% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp.try &.> Time.utc %>
diff --git a/src/invidious/views/components/items_paginated.ecr b/src/invidious/views/components/items_paginated.ecr
new file mode 100644
index 00000000..f69df3fe
--- /dev/null
+++ b/src/invidious/views/components/items_paginated.ecr
@@ -0,0 +1,21 @@
+<%= page_nav_html %>
+
+<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.utc).ago, locale)) %>
- <% elsif Time.utc - item.published > 1.minute %> + <% elsif item.responds_to?(:published) && (Time.utc - item.published) > 1.minute %><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% end %>
+ <%- items.each do |item| -%>
+ <%= rendered "components/item" %>
+ <%- end -%>
+
+
+<%= page_nav_html %>
+
+
+
+
diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr
index 206ba380..af352102 100644
--- a/src/invidious/views/components/player.ecr
+++ b/src/invidious/views/components/player.ecr
@@ -1,18 +1,47 @@