fork from scoresaber-reloaded

src/network/queues/accsaber/api-queue.js

@@ -0,0 +1,33 @@
+import {default as createQueue, PRIORITY} from '../http-queue';
+import {substituteVars} from "../../../utils/format";
+
+const ACCSABER_API_URL = 'https://api.accsaber.com';
+const CATEGORIES_URL = ACCSABER_API_URL + '/categories';
+const RANKING_URL = ACCSABER_API_URL + '/categories/${category}/standings';
+const PLAYER_SCORES_URL = ACCSABER_API_URL + '/players/${playerId}/scores';
+const PLAYER_RANK_HISTORY = ACCSABER_API_URL + '/players/${playerId}/recent-rank-history'
+const LEADERBOARD_URL = ACCSABER_API_URL + '/map-leaderboards/${leaderboardId}';
+const LEADERBOARD_INFO_URL = ACCSABER_API_URL + '/ranked-maps/${leaderboardId}';
+
+export default (options = {}) => {
+  const queue = createQueue(options);
+
+  const {fetchJson, fetchHtml, ...queueToReturn} = queue;
+
+  const categories = async (priority = PRIORITY.FG_LOW, options = {}) => fetchJson(CATEGORIES_URL, options, priority)
+  const ranking = async (category = 'overall', page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(RANKING_URL, {category, page}), options, priority)
+  const scores = async (playerId, page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(PLAYER_SCORES_URL, {playerId, page}), options, priority)
+  const playerRankHistory = async (playerId, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(PLAYER_RANK_HISTORY, {playerId}), options, priority)
+  const leaderboard = async (leaderboardId, page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(LEADERBOARD_URL, {leaderboardId, page}), options, priority)
+  const leaderboardInfo = async (leaderboardId, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(LEADERBOARD_INFO_URL, {leaderboardId}), options, priority)
+
+  return {
+    categories,
+    ranking,
+    scores,
+    playerRankHistory,
+    leaderboard,
+    leaderboardInfo,
+    ...queueToReturn,
+  }
+}

src/network/queues/beatmaps/api-queue.js

@@ -0,0 +1,21 @@
+import {default as createQueue, PRIORITY} from '../http-queue';
+import {substituteVars} from "../../../utils/format";
+
+const BEATMAPS_API_URL = 'https://api.beatsaver.com/';
+const SONG_BY_HASH_URL = BEATMAPS_API_URL + '/maps/hash/${hash}';
+const SONG_BY_KEY_URL = BEATMAPS_API_URL + '/maps/id/${key}'
+
+export default (options = {}) => {
+  const queue = createQueue(options);
+
+  const {fetchJson, fetchHtml, ...queueToReturn} = queue;
+
+  const byHash = async (hash, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(SONG_BY_HASH_URL, {hash}), options, priority)
+  const byKey = async (key, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(SONG_BY_KEY_URL, {key}), options, priority)
+
+  return {
+    byHash,
+    byKey,
+    ...queueToReturn,
+  }
+}

src/network/queues/beatsavior/api-queue.js

@@ -0,0 +1,18 @@
+import {default as createQueue, PRIORITY} from '../http-queue';
+import {substituteVars} from "../../../utils/format";
+
+const BEATSAVIOR_API_URL = '/cors/beat-savior';
+const PLAYER_URL = BEATSAVIOR_API_URL + '/${playerId}';
+
+export default (options = {}) => {
+  const queue = createQueue(options);
+
+  const {fetchJson, fetchHtml, ...queueToReturn} = queue;
+
+  const player = async (playerId, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(PLAYER_URL, {playerId}), options, priority)
+
+  return {
+    player,
+    ...queueToReturn,
+  }
+}

src/network/queues/http-queue.js

@@ -0,0 +1,112 @@
+import {default as createQueue, PRIORITY as QUEUE_PRIORITY} from '../../utils/queue';
+import {SsrError, SsrTimeoutError} from '../../others/errors'
+import {SsrHttpRateLimitError, SsrHttpResponseError, SsrNetworkError, SsrNetworkTimeoutError} from '../errors'
+import {fetchHtml, fetchJson} from '../fetch';
+import makePendingPromisePool from '../../utils/pending-promises'
+import {AbortError} from '../../utils/promise'
+
+const DEFAULT_RETRIES = 2;
+
+export const PRIORITY = {
+  FG_HIGH: QUEUE_PRIORITY.HIGHEST,
+  FG_LOW: QUEUE_PRIORITY.HIGH,
+  BG_HIGH: QUEUE_PRIORITY.NORMAL,
+  BG_NORMAL: QUEUE_PRIORITY.LOW,
+  BG_LOW: QUEUE_PRIORITY.LOWEST,
+}
+
+const resolvePromiseOrWaitForPending = makePendingPromisePool();
+
+export default (options = {}) => {
+  const {retries, rateLimitTick, ...queueOptions} = {retries: DEFAULT_RETRIES, rateLimitTick: 500, ...options};
+  const queue = createQueue(queueOptions);
+
+  const {add, emitter, ...queueToReturn} = queue;
+
+  let lastRateLimitError = null;
+  let rateLimitTimerId = null;
+  let currentRateLimit = {waiting: 0, remaining: null, limit: null, resetAt: null};
+
+  const rateLimitTicker = () => {
+    const expiresInMs = lastRateLimitError && lastRateLimitError.resetAt ? lastRateLimitError.resetAt - new Date() + 1000 : 0;
+    if (expiresInMs <= 0) {
+      emitter.emit('waiting', {waiting: 0, remaining: null, limit: null, resetAt: null});
+
+      if (rateLimitTimerId) clearTimeout(rateLimitTimerId);
+
+      return;
+    }
+
+    const {remaining, limit, resetAt} = lastRateLimitError;
+    emitter.emit('waiting', {waiting: expiresInMs, remaining, limit, resetAt});
+
+    if (rateLimitTimerId) clearTimeout(rateLimitTimerId);
+    rateLimitTimerId = setTimeout(rateLimitTicker, rateLimitTick);
+  }
+
+  const retriedFetch = async (fetchFunc, url, options, priority = PRIORITY.FG_LOW) => {
+    for (let i = 0; i <= retries; i++) {
+      try {
+        return await add(async () => {
+            if (lastRateLimitError) {
+              await lastRateLimitError.waitBeforeRetry();
+
+              lastRateLimitError = null;
+            }
+
+            return fetchFunc(url, options)
+              .then(response => {
+                currentRateLimit = {...response.rateLimit, waiting: 0};
+
+                return response;
+              })
+              .catch(err => {
+                if (err instanceof SsrTimeoutError) throw new SsrNetworkTimeoutError(err.timeout);
+
+                throw err;
+              })
+          },
+          priority,
+        )
+      } catch (err) {
+        if (err instanceof SsrHttpResponseError) {
+          const {remaining, limit, resetAt} = err;
+          currentRateLimit = {waiting: 0, remaining, limit, resetAt};
+        }
+
+        if (err instanceof SsrNetworkError) {
+          const shouldRetry = err.shouldRetry();
+          if (!shouldRetry || i === retries) throw err;
+
+          if (err instanceof SsrHttpRateLimitError) {
+            if (err.remaining <= 0 && err.resetAt && (!lastRateLimitError || !lastRateLimitError.resetAt || lastRateLimitError.resetAt < err.resetAt)) {
+              lastRateLimitError = err;
+
+              rateLimitTicker();
+            }
+          } else {
+            lastRateLimitError = null;
+          }
+        } else if (!(err instanceof DOMException)) {
+          throw err;
+        } else {
+          throw AbortError();
+        }
+      }
+    }
+
+    throw new SsrError('Unknown error');
+  }
+
+  const queuedFetchJson = async (url, options, priority = PRIORITY.FG_LOW) => resolvePromiseOrWaitForPending(url, () => retriedFetch(fetchJson, url, options, priority));
+  const queuedFetchHtml = async (url, options, priority = PRIORITY.FG_LOW) => resolvePromiseOrWaitForPending(url, () => retriedFetch(fetchHtml, url, options, priority));
+
+  const getRateLimit = () => currentRateLimit;
+
+  return {
+    fetchJson: queuedFetchJson,
+    fetchHtml: queuedFetchHtml,
+    getRateLimit,
+    ...queueToReturn,
+  }
+}

src/network/queues/queues.js

@@ -0,0 +1,55 @@
+import {writable} from 'svelte/store'
+import {PRIORITY} from './http-queue'
+import createScoreSaberApiQueue from './scoresaber/api-queue'
+import createScoreSaberPageQueue from './scoresaber/page-queue'
+import createBeatMapsApiQueue from './beatmaps/api-queue'
+import createBeatSaviorApiQueue from './beatsavior/api-queue'
+import createTwitchApiQueue from './twitch/api-queue'
+import createAccSaberApiQueue from './accsaber/api-queue'
+
+export const getResponseBody = response => response ? response.body : null;
+export const isResponseCached = response => !!(response && response.cached)
+export const updateResponseBody = (response, body) => response ? {...response, body} : null;
+
+const initQueue = queue => {
+  let queueState = {
+    size: 0,
+    pending: 0,
+    rateLimit: {waiting: 0, remaining: null, limit: null, resetAt: null},
+    progress: {num: 0, count: 0, progress: 1},
+  };
+
+  const {subscribe, set} = writable(queueState);
+
+  queue.on('change', ({size, pending}) => {
+    const {rateLimit: {waiting}} = queueState;
+    const {remaining, limit, resetAt} = queue.getRateLimit();
+    queueState = {...queueState, size, pending, rateLimit: {waiting, remaining, limit, resetAt}};
+    set(queueState);
+  });
+  queue.on('progress', ({progress, num, count}) => {
+    const {rateLimit: {waiting}} = queueState;
+    const {remaining, limit, resetAt} = queue.getRateLimit();
+    queueState = {...queueState, progress: {num, count, progress}, rateLimit: {waiting, remaining, limit, resetAt}}
+    set(queueState);
+  });
+  queue.on('waiting', ({waiting, remaining, limit, resetAt}) => {
+    queueState = {...queueState, rateLimit: {waiting, remaining, limit, resetAt}}
+    set(queueState);
+  })
+
+  return {
+    subscribe,
+    ...queue,
+  }
+}
+
+export default {
+  SCORESABER_API: initQueue(createScoreSaberApiQueue({concurrency: 3, timeout: 95000})),
+  SCORESABER_PAGE: initQueue(createScoreSaberPageQueue({concurrency: 3, timeout: 30000})),
+  BEATMAPS: initQueue(createBeatMapsApiQueue({concurrency: 1, timeout: 10000, intervalCap: 10, interval: 1000})),
+  BEATSAVIOR: initQueue(createBeatSaviorApiQueue({concurrency: 1, timeout: 10000, intervalCap: 60, interval: 60000})),
+  TWITCH: initQueue(createTwitchApiQueue({concurrency: 8, timeout: 8000, intervalCap: 800, interval: 60000})),
+  ACCSABER: initQueue(createAccSaberApiQueue({concurrency: 2, timeout: 10000})),
+  PRIORITY,
+}

src/network/queues/scoresaber/api-queue.js

@@ -0,0 +1,46 @@
+import {default as createQueue, PRIORITY} from '../http-queue';
+import {substituteVars} from '../../../utils/format'
+import {PLAYER_SCORES_PER_PAGE, PLAYERS_PER_PAGE} from '../../../utils/scoresaber/consts'
+
+export const SS_API_HOST = 'https://new.scoresaber.com';
+export const SS_API_URL = `${SS_API_HOST}/api`;
+
+export const SS_API_PLAYER_INFO_URL = SS_API_URL + '/player/${playerId}/full';
+export const SS_API_RECENT_SCORES_URL = SS_API_URL + '/player/${playerId}/scores/recent/${page}';
+export const SS_API_TOP_SCORES_URL = SS_API_URL + '/player/${playerId}/scores/top/${page}';
+export const SS_API_FIND_PLAYER_URL = SS_API_URL + '/players/by-name/${query}'
+export const SS_API_RANKING_GLOBAL_URL = SS_API_URL + '/players/${page}'
+export const SS_API_RANKING_GLOBAL_PAGES_URL = SS_API_URL + '/players/pages'
+
+export default (options = {}) => {
+  const queue = createQueue(options);
+
+  const {fetchJson, fetchHtml, ...queueToReturn} = queue;
+
+  const fetchScores = async (baseUrl, playerId, page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(baseUrl, {playerId, page}), options, priority);
+
+  const player = async (playerId, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(SS_API_PLAYER_INFO_URL, {playerId}), options, priority);
+
+  const recentScores = async (playerId, page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchScores(SS_API_RECENT_SCORES_URL, playerId, page, priority, options);
+
+  const topScores = async (playerId, page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchScores(SS_API_TOP_SCORES_URL, playerId, page, priority, options);
+
+  const findPlayer = async (query, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(SS_API_FIND_PLAYER_URL, {query: encodeURIComponent(query)}), options, priority);
+
+  const rankingGlobal = async (page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(substituteVars(SS_API_RANKING_GLOBAL_URL, {page}), options, priority);
+
+  const rankingGlobalPages = async (priority = PRIORITY.FG_LOW, options = {}) => fetchJson(SS_API_RANKING_GLOBAL_PAGES_URL, options, priority);
+
+  return {
+    player,
+    findPlayer,
+    rankingGlobal,
+    rankingGlobalPages,
+    recentScores,
+    topScores,
+    SS_API_HOST,
+    PLAYER_SCORES_PER_PAGE,
+    PLAYERS_PER_PAGE,
+    ...queueToReturn,
+  }
+}

src/network/queues/scoresaber/page-queue.js

@@ -0,0 +1,505 @@
+import {default as createQueue, PRIORITY} from '../http-queue';
+import {substituteVars} from '../../../utils/format'
+import {extractDiffAndType} from '../../../utils/scoresaber/format'
+import cfDecryptEmail from '../../../utils/cf-email-decrypt'
+import {capitalize, getFirstRegexpMatch, opt} from '../../../utils/js'
+import {dateFromString} from '../../../utils/date'
+import {LEADERBOARD_SCORES_PER_PAGE} from '../../../utils/scoresaber/consts'
+
+export const SS_HOST = 'https://scoresaber.com';
+const SS_CORS_HOST = '/cors/score-saber';
+const RANKEDS_URL = SS_CORS_HOST + '/api.php?function=get-leaderboards&cat=1&limit=5000&ranked=1&page=${page}';
+const PLAYER_PROFILE_URL = SS_CORS_HOST + '/u/${playerId}?page=1&sort=2'
+const COUNTRY_RANKING_URL = SS_CORS_HOST + '/global/${page}?country=${country}'
+const LEADERBOARD_URL = SS_CORS_HOST + '/leaderboard/${leaderboardId}?page=${page}'
+
+export const parseSsInt = text => {
+  const value = getFirstRegexpMatch(/(-?[0-9,]+)\s*$/, text)
+  return value ? parseInt(value.replace(/[^\d-]/g, '') , 10) : null;
+}
+export const parseSsFloat = text => text ? parseFloat(getFirstRegexpMatch(/([0-9,.]+)\s*$/, text.replace(/[^\d.]/g, ''))) : null;
+
+export default (options = {}) => {
+  const queue = createQueue(options);
+
+  const {fetchJson, fetchHtml, ...queueToReturn} = queue;
+
+  const processRankeds = (data) => {
+    if (!data || !data.songs || !Array.isArray(data.songs)) return null;
+
+    return data.songs.map(s => {
+      const {
+        uid: leaderboardId,
+        id: hash,
+        name,
+        songSubName: subName,
+        songAuthorName: authorName,
+        levelAuthorName,
+        stars,
+        image: imageUrl,
+        diff
+      } = s;
+
+      const diffInfo = extractDiffAndType(diff);
+
+      return {leaderboardId, hash, name, subName, authorName, levelAuthorName, imageUrl, stars, diff, diffInfo};
+    })
+  }
+
+  const getImgUrl = imgUrl => {
+    try {
+      const aUrl = new URL(imgUrl);
+      return SS_HOST + aUrl.pathname;
+    }
+    catch(err) {
+      return null;
+    }
+  }
+
+  const rankeds = async (page = 1, priority = PRIORITY.BG_NORMAL, options = {}) => fetchJson(substituteVars(RANKEDS_URL, {page}), options, priority)
+    .then(r => {
+      r.body = processRankeds(r.body);
+
+      return r;
+    })
+
+  const processPlayerProfile = (playerId, doc) => {
+    cfDecryptEmail(doc);
+
+    let avatar = getImgUrl(opt(doc.querySelector('.column.avatar img'), 'src', null));
+
+    let playerName = opt(doc.querySelector('.content .column:not(.avatar) .title a'), 'innerText');
+    playerName = playerName ? playerName.trim() : null;
+
+    let country = getFirstRegexpMatch(/^.*?\/flags\/([^.]+)\..*$/, opt(doc.querySelector('.content .column .title img'), 'src'));
+    country = country ? country.toUpperCase() : null;
+
+    let pageNum = parseSsInt(opt(doc.querySelector('.pagination .pagination-list li a.is-current'), 'innerText', null));
+    pageNum = !isNaN(pageNum) ? pageNum : null
+
+    let pageQty = parseSsInt(opt(doc.querySelector('.pagination .pagination-list li:last-of-type'), 'innerText', null));
+    pageQty = !isNaN(pageQty) ? pageQty : null
+
+    let totalItems = parseSsFloat(getFirstRegexpMatch(/^\s*<strong>(?:[^:]+)\s*:?\s*<\/strong>\s*(.*)$/, opt(doc.querySelector('.columns .column:not(.is-narrow) ul li:nth-of-type(3)'), 'innerHTML')))
+    totalItems = !isNaN(totalItems) ? totalItems : 0;
+
+    let playerRank = parseSsInt(opt(doc.querySelector('.content .column ul li:first-of-type a:first-of-type'), 'innerText'));
+    playerRank = !isNaN(playerRank) ? playerRank : null;
+
+    let countryRank = parseSsInt(opt(doc.querySelector('.content .column ul li:first-of-type a[href^="/global?country="]'), 'innerText'))
+    countryRank = !isNaN(countryRank) ? countryRank : null;
+
+    const stats = [{key: 'Player ranking', type: 'rank', value: playerRank, countryRank: countryRank}]
+      .concat(
+        [...doc.querySelectorAll('.content .column ul li')]
+          .map(li => {
+            const matches = li.innerHTML.match(/^\s*<strong>([^:]+)\s*:?\s*<\/strong>\s*(.*)$/);
+            if (!matches) return null;
+
+            const mapping = [
+              {key: 'Performance Points', type: 'number', precision: 2, suffix: 'pp', number: true,},
+              {key: 'Play Count', type: 'number', precision: 0, number: true, colorVar: 'selected',},
+              {key: 'Total Score', type: 'number', precision: 0, number: true, colorVar: 'selected',},
+              {
+                key: 'Replays Watched by Others',
+                type: 'number',
+                precision: 0,
+                title: 'profile.stats.replays',
+                number: true,
+                colorVar: 'dimmed',
+              },
+              {key: 'Role', number: false, colorVar: 'dimmed'},
+              {key: 'Inactive Account', number: false, colorVar: 'decrease'},
+              {key: 'Banned', number: false, colorVar: 'decrease'},
+            ];
+
+            const value = mapping.filter(m => m.number).map(m => m.key).includes(matches[1])
+              ? parseSsFloat(matches[2])
+              : matches[2];
+
+            const item = mapping.find(m => m.key === matches[1]);
+            return item ? {...item, value} : {label: matches[1], value};
+          })
+          .filter(s => s)
+      ).reduce((cum, item) => {
+        if (item.key)
+          switch (item.key) {
+            case 'Player ranking':
+              cum.rank = item.value;
+              cum.countryRank = item.countryRank;
+              break;
+
+            case 'Performance Points':
+              cum.pp = item.value;
+              break;
+            case 'Play Count':
+              cum.playCount = item.value;
+              break;
+            case 'Total Score':
+              cum.totalScore = item.value;
+              break;
+            case 'Replays Watched by Others':
+              cum.replays = item.value;
+              break;
+            case 'Role':
+              cum.role = item.value;
+              break;
+            case 'Inactive Account':
+              cum.inactiveAccount = true;
+              break;
+            case 'Banned':
+              cum.bannedAccount = true;
+              break;
+          }
+
+        return cum;
+      }, {inactiveAccount: false, bannedAccount: false});
+
+    const scores = [...doc.querySelectorAll('table.ranking tbody tr')].map(tr => {
+      let ret = {lastUpdated: new Date()};
+
+      const rank = tr.querySelector('th.rank');
+      if (rank) {
+        const rankMatch = parseSsInt(rank.innerText);
+        ret.rank = !isNaN(rankMatch) ? rankMatch : null;
+      } else {
+        ret.rank = null;
+      }
+
+      const song = tr.querySelector('th.song a');
+      if (song) {
+        const leaderboardId = parseInt(getFirstRegexpMatch(/leaderboard\/(\d+)/, song.href), 10);
+        ret.leaderboardId = leaderboardId ? leaderboardId : null;
+      } else {
+        ret.leaderboardId = null;
+      }
+
+      const img = tr.querySelector('th.song img');
+      const imgMatch = img ? img.src.match(/([^\/]+)\.(jpg|jpeg|png)$/) : null;
+      ret.songHash = imgMatch ? imgMatch[1] : null;
+
+      const songPp = tr.querySelector('th.song a .songTop.pp');
+      const songMatch = songPp
+        ? songPp.innerHTML
+          .replace(/&amp;/g, '&')
+          .replace(/<span class="__cf_email__" data-cfemail="[^"]+">\[email&nbsp;protected]<\/span>/g, '')
+          .match(/^(.*?)\s*<span[^>]+>(.*?)<\/span>/)
+        : null;
+      if (songMatch) {
+        const songAuthorMatch = songMatch[1].match(/^(.*?)\s-\s(.*)$/);
+        if (songAuthorMatch) {
+          ret.songName = songAuthorMatch[2];
+          ret.songSubName = '';
+          ret.songAuthorName = songAuthorMatch[1];
+        } else {
+          ret.songName = songMatch[1];
+          ret.songSubName = '';
+          ret.songAuthorName = '';
+        }
+        ret.difficultyRaw = '_' + songMatch[2].replace('Expert+', 'ExpertPlus') + '_SoloStandard'
+      } else {
+        ret = Object.assign(ret, {songName: null, songSubName: null, songAuthorName: null, difficultyRaw: null});
+      }
+
+      const songMapper = tr.querySelector('th.song a .songTop.mapper');
+      ret.levelAuthorName = songMapper ? songMapper.innerText : null;
+
+      const songDate = tr.querySelector('th.song span.songBottom.time');
+      ret.timeSet = songDate ? dateFromString(songDate.title) : null;
+
+      const pp = parseSsFloat(opt(tr.querySelector('th.score .scoreTop.ppValue'), 'innerText'));
+      ret.pp = !isNaN(pp) ? pp : null;
+
+      const ppWeighted = parseSsFloat(getFirstRegexpMatch(/^\(([0-9.]+)pp\)$/, opt(tr.querySelector('th.score .scoreTop.ppWeightedValue'), 'innerText')));
+      ret.ppWeighted = !isNaN(ppWeighted) ? ppWeighted : null;
+
+      const scoreInfo = tr.querySelector('th.score .scoreBottom');
+      const scoreInfoMatch = scoreInfo ? scoreInfo.innerText.match(/^([^:]+):\s*([0-9,.]+)(?:.*?\((.*?)\))?/) : null;
+      if (scoreInfoMatch) {
+        switch (scoreInfoMatch[1]) {
+          case "score":
+            ret.acc = null;
+            scoreInfoMatch[3] = scoreInfoMatch[3] ? scoreInfoMatch[3].replace('-','').trim() : null
+            ret.mods = scoreInfoMatch[3] && scoreInfoMatch[3].length ? scoreInfoMatch[3].split(',').filter(m => m && m.trim().length) : null;
+            ret.score = parseSsFloat(scoreInfoMatch[2]);
+            break;
+
+          case "accuracy":
+            ret.score = null;
+            scoreInfoMatch[3] = scoreInfoMatch[3] ? scoreInfoMatch[3].replace('-','').trim() : null
+            ret.mods = scoreInfoMatch[3] && scoreInfoMatch[3].length ? scoreInfoMatch[3].split(',').filter(m => m && m.trim().length) : null;
+            ret.acc = parseSsFloat(scoreInfoMatch[2]);
+            break;
+        }
+      }
+
+      return ret;
+    });
+    const recentPlay = scores && scores.length && scores[0].timeSet ? scores[0].timeSet : null;
+
+    return {
+      player: {
+        playerInfo: {
+          playerId,
+          playerName,
+          avatar,
+          externalProfileUrl: opt(doc.querySelector('.content .column:not(.avatar) .title a'), 'href', null),
+          history: getFirstRegexpMatch(/data:\s*\[([0-9,]+)\]/, doc.body.innerHTML),
+          country,
+          badges: [...doc.querySelectorAll('.column.avatar center img')].map(img => ({
+            image: getImgUrl(img.src),
+            description: img.title
+          })),
+          rank: stats.rank ? stats.rank : null,
+          countryRank: stats.countryRank ? stats.countryRank : null,
+          pp: stats.pp !== undefined ? stats.pp : null,
+          inactive: stats.inactiveAccount ? 1 : 0,
+          banned: stats.bannedAccount ? 1 : 0,
+          role: '',
+        },
+        scoreStats: {
+          totalScore: stats.totalScore ? stats.totalScore : 0,
+          totalPlayCount: stats.playCount ? stats.playCount : 0,
+          replays: stats.replays ? stats.replays : null,
+        },
+        recentPlay,
+        recentPlayLastUpdated: recentPlay ? new Date() : null,
+      },
+      scores,
+      others: {
+        pageNum,
+        pageQty,
+        totalItems,
+      }
+    };
+  }
+
+  const player = async (playerId, priority = PRIORITY.FG_LOW, options = {}) => fetchHtml(substituteVars(PLAYER_PROFILE_URL, {playerId}), options, priority)
+    .then(r => {
+      r.body = processPlayerProfile(playerId, r.body);
+
+      return r
+    })
+
+  const processCountryRanking = (country, doc) => {
+    cfDecryptEmail(doc);
+
+    const data = [...doc.querySelectorAll('.ranking.global .player a')]
+      .map(a => {
+        const tr = a.closest("tr");
+        const id = getFirstRegexpMatch(/\/(\d+)$/, a.href)
+
+        const avatar = getImgUrl(opt(tr.querySelector('td.picture img'), 'src', null));
+
+        let country = getFirstRegexpMatch(/^.*?\/flags\/([^.]+)\..*$/, opt(tr.querySelector('td.player img'), 'src', null));
+        country = country ? country.toUpperCase() : null;
+
+        let difference = parseSsInt(opt(tr.querySelector('td.diff'), 'innerText', null));
+        difference = !isNaN(difference) ? difference : null
+
+        let playerName = opt(a.querySelector('.songTop.pp'), 'innerText');
+        playerName = playerName || playerName === '' ? playerName.trim() : null;
+
+        let pp = parseSsFloat(opt(tr.querySelector('td.pp .scoreTop.ppValue'), 'innerText'));
+        pp = !isNaN(pp) ? pp : null;
+
+        let rank = parseSsInt(getFirstRegexpMatch(/^\s*#(\d+)\s*$/, opt(tr.querySelector('td.rank'), 'innerText', null)));
+        rank = !isNaN(rank) ? rank : null
+
+        return {
+          avatar,
+          country,
+          difference,
+          history: [],
+          playerId: id,
+          playerName,
+          pp,
+          rank,
+        }
+      })
+
+    return {players: data};
+  }
+
+  const countryRanking = async (country, page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchHtml(substituteVars(COUNTRY_RANKING_URL, {country, page}), options, priority)
+    .then(r => {
+      r.body = processCountryRanking(country, r.body)
+
+      return r;
+    })
+
+  const parseSsLeaderboardScores = doc => {
+    cfDecryptEmail(doc);
+
+    return [...doc.querySelectorAll('table.ranking tbody tr')].map(tr => {
+      let ret = {player: {playerInfo: {countries: []}}, score: {lastUpdated: new Date()}};
+
+      const parseValue = selector => {
+        const val = parseSsFloat(opt(tr.querySelector(selector), 'innerText'));
+
+        return !isNaN(val) ? val : null;
+      }
+
+      ret.player.playerInfo.avatar = getImgUrl(opt(tr.querySelector('.picture img'), 'src', null));
+
+      ret.score.rank = parseSsInt(opt(tr.querySelector('td.rank'), 'innerText'));
+      if (isNaN(ret.score.rank)) ret.score.rank = null;
+
+      const player = tr.querySelector('.player a');
+      if (player) {
+        let country = getFirstRegexpMatch(/^.*?\/flags\/([^.]+)\..*$/, opt(player.querySelector('img'), 'src', ''));
+        country = country ? country.toUpperCase() : null;
+        if (country) {
+          ret.player.playerInfo.country = country
+          ret.player.playerInfo.countries.push({country, rank: null});
+        }
+
+        ret.player.name = opt(player.querySelector('span.songTop.pp'), 'innerText')
+        ret.player.name = ret.player.name ? ret.player.name.trim().replace('&#039;', "'") : null;
+        ret.player.playerId = getFirstRegexpMatch(/\/u\/(\d+)((\?|&|#).*)?$/, opt(player, 'href', ''));
+        ret.player.playerId = ret.player.playerId ? ret.player.playerId.trim() : null;
+      } else {
+        ret.player.playerId = null;
+        ret.player.name = null;
+        ret.player.playerInfo.country = null;
+      }
+
+      ret.score.score = parseValue('td.score');
+
+      ret.score.timeSetString = opt(tr.querySelector('td.timeset'), 'innerText', null);
+      if (ret.score.timeSetString) ret.score.timeSetString = ret.score.timeSetString.trim();
+
+      ret.score.mods = opt(tr.querySelector('td.mods'), 'innerText');
+      ret.score.mods = ret.score.mods ? ret.score.mods.replace('-','').trim() : null
+      ret.score.mods = ret.score.mods && ret.score.mods.length ? ret.score.mods.split(',').filter(m => m && m.trim().length) : null;
+
+      ret.score.pp = parseValue('td.pp .scoreTop.ppValue');
+
+      ret.score.percentage = parseValue('td.percentage');
+
+      return ret;
+    });
+  }
+
+  const processLeaderboard = (leaderboardId, page, doc) => {
+    cfDecryptEmail(doc);
+
+    const diffs = [...doc.querySelectorAll('.tabs ul li a')].map(a => {
+      let leaderboardId = parseInt(getFirstRegexpMatch(/leaderboard\/(\d+)$/, a.href), 10);
+      if (isNaN(leaderboardId)) leaderboardId = null;
+
+      const span = a.querySelector('span');
+      const color = span ? span.style.color : null;
+
+      return {name: a.innerText, leaderboardId, color};
+    });
+
+    const currentDiffHuman = opt(doc.querySelector('.tabs li.is-active a span'), 'innerText', null);
+
+    let diff = null;
+    let diffInfo = null;
+    if (currentDiffHuman) {
+      const lowerCaseDiff = currentDiffHuman.toLowerCase().replace('+', 'Plus');
+      diff = `_${capitalize(lowerCaseDiff)}_SoloStandard`;
+      diffInfo = {type: 'Standard', diff: lowerCaseDiff}
+    }
+
+    const songName = opt(doc.querySelector('.column.is-one-third-desktop .box:first-of-type .title a'), 'innerText', null);
+
+    const imageUrl = getImgUrl(opt(doc.querySelector('.column.is-one-third-desktop .box:first-of-type .columns .column.is-one-quarter img'), 'src', null));
+
+    const songInfo = [
+      {id: 'hash', label: 'ID', value: null},
+      {id: 'scores', label: 'Scores', value: null},
+      {id: 'status', label: 'Status', value: null},
+      {id: 'totalScores', label: 'Total Scores', value: null},
+      {id: 'notes', label: 'Note Count', value: null},
+      {id: 'bpm', label: 'BPM', value: null},
+      {id: 'stars', label: 'Star Difficulty', value: null},
+      {id: 'levelAuthorName', label: 'Mapped by', value: null},
+    ]
+      .map(sid => {
+        let songInfoBox = doc.querySelector('.column.is-one-third-desktop .box:first-of-type')
+        return {
+          ...sid,
+          value: songInfoBox ? songInfoBox.innerHTML.match(new RegExp(sid.label + ':\\s*<b>(.*?)</b>', 'i')) : null,
+        }
+      })
+      .concat([{id: 'name', value: [null, songName]}])
+      .reduce((cum, sid) => {
+        let value = Array.isArray(sid.value) ? sid.value[1] : null;
+
+        if (value !== null && ['scores', 'totalScores', 'bpm', 'notes'].includes(sid.id)) {
+          value = parseSsFloat(value);
+
+          if (value !== null) {
+            cum.stats[sid.id] = value;
+          }
+
+          return cum;
+        }
+        if (value !== null && sid.id === 'stars') value = parseSsFloat(value);
+        if (value && sid.id === 'name') {
+          const songAuthorMatch = value.match(/^(.*?)\s-\s(.*)$/);
+          if (songAuthorMatch) {
+            value = songAuthorMatch[2];
+            cum.authorName = songAuthorMatch[1];
+          } else {
+            cum.authorName = '';
+          }
+          cum.subName = '';
+        }
+        if (value && sid.id === 'levelAuthorName') {
+          const el = doc.createElement('div');
+          el.innerHTML = value;
+          value = el.innerText;
+        }
+        if (value && sid.id === 'status') {
+          cum.stats[sid.id] = value;
+          return cum;
+        }
+        if (value !== null) cum[sid.id] = value;
+
+        return cum;
+      }, {imageUrl, stats: {}});
+
+    const {stats, ...song} = songInfo;
+    const leaderboard = {leaderboardId, song, diffInfo, stats};
+
+    let pageQty = parseInt(opt(doc.querySelector('.pagination .pagination-list li:last-of-type'), 'innerText', null), 10)
+    if (isNaN(pageQty)) pageQty = null;
+
+    let scoresQty = opt(stats, 'scores', 0);
+    if (isNaN(scoresQty)) scoresQty = null;
+
+    const totalItems = pageQty && scoresQty ? (Math.ceil(scoresQty / LEADERBOARD_SCORES_PER_PAGE) > pageQty ? pageQty * LEADERBOARD_SCORES_PER_PAGE : scoresQty) : null;
+
+    let diffChartText = getFirstRegexpMatch(/'difficulty',\s*([0-9.,\s]+)\s*\]/, doc.body.innerHTML)
+    let diffChart = (diffChartText ? diffChartText : '').split(',').map(i => parseFloat(i)).filter(i => i && !isNaN(i));
+
+    return {
+      diffs,
+      leaderboard,
+      diffChart,
+      page,
+      pageQty,
+      totalItems,
+      scores: parseSsLeaderboardScores(doc),
+    }
+  }
+
+  const leaderboard = async (leaderboardId, page = 1, priority = PRIORITY.FG_LOW, options = {}) => fetchHtml(substituteVars(LEADERBOARD_URL, {leaderboardId, page}), options, priority)
+    .then(r => {
+      r.body = processLeaderboard(leaderboardId, page, r.body);
+
+      return r;
+    })
+
+  return {
+    rankeds,
+    player,
+    countryRanking,
+    leaderboard,
+    ...queueToReturn,
+  }
+}

src/network/queues/twitch/api-queue.js

@@ -0,0 +1,51 @@
+import {default as createQueue, PRIORITY} from '../http-queue';
+import ssrConfig from '../../../ssr-config'
+import {substituteVars} from "../../../utils/format";
+
+const CLIENT_ID = 'u0swxz56n4iumc634at1osoqdk31qt';
+
+const TWITCH_AUTH_URL = 'https://id.twitch.tv/oauth2'
+const AUTHORIZATION_URL = `${TWITCH_AUTH_URL}/authorize?client_id=${CLIENT_ID}&redirect_uri=${encodeURIComponent(ssrConfig.domain + '/twitch')}&response_type=token` + '&scope=${scopes}&state=${state}';
+const VALIDATE_URL = `${TWITCH_AUTH_URL}/validate`
+
+const TWITCH_API_URL = 'https://api.twitch.tv/helix';
+const PROFILE_URL = TWITCH_API_URL + '/users?login=${login}';
+const VIDEOS_URL = TWITCH_API_URL + '/videos?user_id=${userId}&type=${type}&first=100';
+const STREAMS_URL = TWITCH_API_URL + '/streams?user_id=${userId}';
+
+export default (options = {}) => {
+  const queue = createQueue(options);
+
+  const {fetchJson, fetchHtml, ...queueToReturn} = queue;
+
+  const fetchApi = (url, accessToken, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(
+    url,
+    {
+      ...options,
+      headers: {
+        'Client-ID': CLIENT_ID,
+        'Authorization': `Bearer ${accessToken}`
+      }
+    },
+    priority,
+  )
+
+  const getAuthUrl = (state = '', scopes = '') => substituteVars(AUTHORIZATION_URL, {state: encodeURIComponent(state), scopes: encodeURIComponent(scopes)});
+
+  const validateToken = async (accessToken, priority = PRIORITY.FG_LOW, options = {}) => fetchJson(VALIDATE_URL, {...options, headers: {'Authorization': `OAuth ${accessToken}`}}, priority)
+
+  const profile = async (accessToken, login, priority = PRIORITY.FG_LOW, options = {}) => fetchApi(substituteVars(PROFILE_URL, {login: encodeURIComponent(login)}), accessToken, priority, options)
+
+  const videos = async (accessToken, userId, type = 'archive', priority = PRIORITY.FG_LOW, options = {}) => fetchApi(substituteVars(VIDEOS_URL, {userId: encodeURIComponent(userId), type: encodeURIComponent(type)}), accessToken, priority, options)
+
+  const streams = async (accessToken, userId, priority = PRIORITY.FG_LOW, options = {}) => fetchApi(substituteVars(STREAMS_URL, {userId: encodeURIComponent(userId)}), accessToken, priority, options)
+
+  return {
+    getAuthUrl,
+    validateToken,
+    profile,
+    videos,
+    streams,
+    ...queueToReturn,
+  }
+}