Merge pull request #471 from FoxxMD/componentPagination

feat: Implement pagination fetching for component listen data

Author: FoxxMD

package-lock.json

@@ -52,6 +52,7 @@
     "@awaitjs/express": "^0.6.3",
     "@curvenote/ansi-to-react": "^7.0.0",
     "@donedeal0/superdiff": "^1.1.1",
+    "@ewanc26/tid": "^1.0.2",
     "@fortawesome/fontawesome-svg-core": "^6.4.2",
     "@fortawesome/free-solid-svg-icons": "^6.4.2",
     "@fortawesome/react-fontawesome": "^0.2.0",
@@ -120,6 +121,7 @@
     "ntfy": "^1.7.6",
     "p-event": "^4.2.0",
     "p-map": "^7.0.4",
+    "p-retry": "^7.1.1",
     "passport": "^0.6.0",
     "passport-deezer": "^0.2.0",
     "patch-package": "^8.0.0",
@@ -174,6 +176,7 @@
     "@types/react": "^18.2.18",
     "@types/react-dom": "^18.2.7",
     "@types/react-window": "^1.8.5",
+    "@types/sinon": "^21.0.0",
     "@types/spotify-web-api-node": "^5.0.7",
     "@types/superagent": "^8.1.9",
     "@types/xml2js": "^0.4.11",
@@ -185,8 +188,9 @@
     "git-cliff": "^2.12.0",
     "mocha": "^10.3.0",
     "mockdate": "^3.0.5",
-    "msw": "^2.1.2",
+    "msw": "^2.12.10",
     "nodemon": "^3.0.3",
+    "sinon": "^21.0.2",
     "ts-essentials": "^9.1.2",
     "typescript": "5.5.4",
     "typescript-eslint": "^7.0.1",

package.json

@@ -17,7 +17,7 @@ import { fileOrDirectoryIsWriteable } from '../utils.js';
 import { asCacheAuthProvider, asCacheMetadataProvider, asCacheScrobbleProvider, CacheAuthProvider, CacheConfig, CacheConfigOptions, CacheMetadataProvider, CacheProvider, CacheScrobbleProvider } from './infrastructure/Atomic.js';
 import { Typeson } from 'typeson';
 import { builtin } from 'typeson-registry';
-import { MaybeLogger } from './logging.js';
+import { loggerNoop } from './logging.js';
 import { ListenProgressPositional, ListenProgressTS } from '../sources/PlayerState/ListenProgress.js';
 const configDir = process.env.CONFIG_DIR || path.resolve(projectDir, `./config`);
 
@@ -50,6 +50,7 @@ export class MSCache {
     cacheAuth: Cacheable;
     regexCache: ReturnType<typeof cacheFunctions>;
     cacheTransform: Cacheable;
+    cacheClientScrobbles: Cacheable;
 
     logger: Logger;
 
@@ -96,6 +97,7 @@ export class MSCache {
 
         this.regexCache = cacheFunctions(this.config.regex);
         this.cacheTransform = new Cacheable({primary: initMemoryCache({lruSize: 500})});
+        this.cacheClientScrobbles = new Cacheable({primary: initMemoryCache({lruSize: 100, ttl: '5m'})});
     }
 
     init = async () => {
@@ -213,7 +215,7 @@ export const flatCacheCreate = (opts: FlatCacheOptions) => {
     });
 }
 
-export const flatCacheLoad = async (flatCache: FlatCache, logger: MaybeLogger): Promise<void> => {
+export const flatCacheLoad = async (flatCache: FlatCache, logger: Logger = loggerNoop): Promise<void> => {
 
     const cachePath = path.join(flatCache.cacheDir, flatCache.cacheId);
     try {
@@ -224,7 +226,7 @@ export const flatCacheLoad = async (flatCache: FlatCache, logger: MaybeLogger):
 
     const streamPromise = new Promise((resolve, reject) => {
         flatCache.loadFileStream(cachePath, (progress: number, total: number) => {
-            logger.debug(`Loading ${progress}/${total} chunks...`);
+            logger.trace(`Loading ${progress}/${total} chunks...`);
         }, () => {
             resolve(true);
         }, (err: Error) => {
@@ -260,7 +262,7 @@ export const flatCacheLoad = async (flatCache: FlatCache, logger: MaybeLogger):
     }
 }
 
-export const initFileCache = async (opts: FlatCacheOptions = {}, logger: MaybeLogger = new MaybeLogger()): Promise<[Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined]> => {
+export const initFileCache = async (opts: FlatCacheOptions = {}, logger: Logger = loggerNoop): Promise<[Keyv | KeyvStoreAdapter | undefined, FlatCache | undefined]> => {
     const flatCache = flatCacheCreate(opts);
     try {
         await flatCacheLoad(flatCache, logger);

src/backend/common/Cache.ts

@@ -1,9 +1,10 @@
 import { Logger } from '@foxxmd/logging';
-import { Dayjs } from "dayjs";
+import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core";
+import { Dayjs, ManipulateType } from "dayjs";
 import { Request, Response } from "express";
 import { NextFunction, ParamsDictionary, Query } from "express-serve-static-core";
 import { FixedSizeList } from 'fixed-size-list';
-import { isPlayObject, PlayMeta, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless } from "../../../core/Atomic.js";
+import { isPlayObject, PlayMeta, PlayMetaLifecycleless, PlayObject, PlayObjectLifecycleless, UnixTimestamp } from "../../../core/Atomic.js";
 import TupleMap from "../TupleMap.js";
 import { MusicBrainzApi } from 'musicbrainz-api';
 
@@ -346,6 +347,10 @@ export interface Authenticatable {
     testAuth: () => Promise<any>
 }
 
+export interface ScrobbleRangeFetchable {
+    getScrobblesForTimeRange: TimeRangeListensFetcher
+}
+
 export interface MdnsDeviceInfo {
     name: string
     type: string
@@ -406,6 +411,8 @@ export const MBID_VARIOUS_ARTISTS = "89ad4ac3-39f7-470e-963a-56509c546377";
 
 export type MusicBrainzSingletonMap = Map<string,MusicBrainzApi>;
 
+
+
 /* https://websocket.org/reference/close-codes/ */
 export const WEBSOCKET_CLOSE_CODE_REASONS = {
     1000: 'Closed gracefully',
@@ -428,4 +435,83 @@ export const WEBSOCKET_CLOSE_CODE = {
     InternalError: 1011
 }
 
-export const WEBSOCKET_CLOSE_CODES_RETRY = [1006,1011,1001];
+export const WEBSOCKET_CLOSE_CODES_RETRY = [1006,1011,1001];export interface PaginatedLimit {
+    /** per page max number of results to return */
+    limit?: number
+}
+
+export interface PaginatedTimeRangeOptions {
+    /** Unix timestamp */
+    from: UnixTimestamp
+    /** Unix timestamp */
+    to: UnixTimestamp
+    /** maximum number of results to fetch before returning early */
+    fetchMax?: number
+}
+
+export type PaginatedTimeRangeCommonOptions = Partial<PaginatedTimeRangeOptions> & PaginatedLimit;
+
+export type CursorType = number | string;
+
+export interface PaginatedListensOptions<T extends CursorType> extends PaginatedLimit {
+    cursor: T
+}
+
+export interface PagelessListensTimeRangeOptions extends PaginatedTimeRangeCommonOptions {
+}
+
+export interface PaginatedListensTimeRangeOptions<T extends CursorType = CursorType> extends Partial<PaginatedTimeRangeOptions>, PaginatedListensOptions<T> {
+}
+
+export interface PaginatedResults<T extends CursorType = CursorType> {
+    total?: number
+    more?: boolean
+    cursorNext?: T
+    order?: 'desc' | 'asc'
+}
+
+export interface PaginatedListens {
+    getPaginatedListens(params: PaginatedListensOptions<CursorType>): Promise<{data: PlayObject[], meta: PaginatedListensOptions<CursorType> & PaginatedResults<CursorType>}>
+}
+
+export const hasPaginagedListens = (obj: Object): obj is PaginatedListens => {
+    return 'getPaginatedListens' in obj;
+}
+
+export interface PaginatedTimeRangeListensResult<T extends CursorType> {
+    data: PlayObject[];
+    meta: PaginatedListensTimeRangeOptions<T> & PaginatedResults<T>;
+}
+export interface PaginatedTimeRangeListens<T extends CursorType = CursorType> {
+    getPaginatedTimeRangeListens(params: PaginatedListensTimeRangeOptions<T>): Promise<PaginatedTimeRangeListensResult<T>>
+    getPaginatedUnitOfTime(): ManipulateType;
+}
+
+export const hasPaginatedTimeRangeListens = (obj: Object): obj is PaginatedTimeRangeListens => {
+    return 'getPaginatedTimeRangeListens' in obj;
+}
+
+export interface PagelessTimeRangeListensResult {
+    data: PlayObject[]
+    meta: PagelessListensTimeRangeOptions & PaginatedResults
+}
+export interface PagelessTimeRangeListens {
+    getPagelessTimeRangeListens(params: PagelessListensTimeRangeOptions): Promise<PagelessTimeRangeListensResult>
+    getPaginatedUnitOfTime(): ManipulateType;
+}
+
+export const hasPagelessTimeRangeListens = (obj: Object): obj is PagelessTimeRangeListens => {
+    return 'getPagelessTimeRangeListens' in obj;
+}
+
+export type PaginatedTimeRangeSource = PaginatedTimeRangeListens | PagelessTimeRangeListens;
+export type PaginatedSource = PaginatedListens | PaginatedTimeRangeSource;
+
+export type TimeRangeListensFetcher<T extends CursorType = CursorType> = (opts: PaginatedTimeRangeCommonOptions | PaginatedListensTimeRangeOptions<T>) => Promise<PlayObject[]>
+
+export interface ScrobbleRangeResult {
+    plays: PlayObject[]
+    fetchedAt: Dayjs
+}
+
+export const REFRESH_STALE_DEFAULT = 60;

src/backend/common/infrastructure/Atomic.ts

@@ -1,3 +1,4 @@
+import { UnixTimestamp } from "../../../../../core/Atomic.js"
 import { RequestRetryOptions } from "../common.js"
 import { CommonClientConfig, CommonClientData } from "./index.js"
 
@@ -15,6 +16,18 @@ export interface ListenObjectResponse {
     track: TrackResponse
 }
 
+export interface GetListensOptions {
+    limit?: number
+    page?: number
+    week?: number
+    month?: number
+    year?: number
+
+    // new in 0.1.0
+    to?: UnixTimestamp
+    from?: UnixTimestamp
+}
+
 export interface TrackResponse {
     id: number
     title: string

src/backend/common/infrastructure/config/client/koito.ts

@@ -38,3 +38,11 @@ export interface ListenBrainzClientConfig extends CommonClientConfig {
 export interface ListenBrainzClientAIOConfig extends ListenBrainzClientConfig {
     type: 'listenbrainz'
 }
+
+
+/** https://github.com/metabrainz/listenbrainz-server/pull/2572
+ * https://github.com/metabrainz/listenbrainz-server/blob/master/listenbrainz/webserver/views/api_tools.py#L48
+ */
+export const MAX_ITEMS_PER_GET_LZ = 1000;
+export const DEFAULT_ITEMS_PER_GET_LZ = 25;
+export const DEFAULT_MS_ITEMS_PER_GET_LZ = 100;

src/backend/common/infrastructure/config/client/listenbrainz.ts

@@ -115,3 +115,19 @@ export class MaybeLogger {
         }
     }
 }
+
+const noopLog = (_: any, ...rest: any) => undefined;
+
+export const loggerNoop: Logger = {
+    trace: noopLog,
+    debug: noopLog,
+    log: noopLog,
+    info: noopLog,
+    verbose: noopLog,
+    warn: noopLog,
+    error: noopLog,
+    fatal: noopLog,
+    silent: noopLog,
+    level: 'silent',
+    child: (_: any, ...rest: any) => loggerNoop as Logger
+} as unknown as Logger;