apps/keeper/src/jobs/collections.ts

1// collections: find real Solana NFT collections that can be paired with a coin, and keep their data fresh.2//  1. Catalogue (daily): HowRare.is lists ~1,900 collections with their Magic Eden symbol, size and holders.3//     Candidates: a Magic Eden symbol, 100–20,000 NFTs, the most holders first.4//  2. Market (hourly): Magic Eden floor, listings and 7-day volume per candidate. Traded ones rank by volume.5//  3. Listing (daily per collection): for the top traded candidates, download every NFT with its traits from HowRare,6//     read a sample of Metaplex metadata accounts on-chain for the verified collection key (the collection's address),7//     rank rarity, run every check in core/collections.ts and upsert collections + nfts.8// Collections that already have coins are never dropped from the refresh, whatever their rank.9import { and, eq, inArray, isNotNull, sql } from 'drizzle-orm';10import { createHash } from 'node:crypto';11import { collections, coins, nfts } from '@solary/db';12import { MAX_COLLECTION_SIZE, MIN_COLLECTION_SIZE, checkCollection, rankRarity, slugify } from '@solary/core';13import { event, getSetting, setSetting, type JobCtx, type Stats } from '../ctx';14import type { HowRareListing, MeStats } from '../nft/sources';15 16interface CatalogRow extends HowRareListing {17  floor: string;18  vol7d: string;19  volAll: string;20  listed: number;21  statsAt: number;22  /** last time listing was attempted and the collection could not be listed */23  skipAt?: number;24}25 26const DAY = 86_400_000;27const HOUR = 3_600_000;28 29export async function collectionsJob(ctx: JobCtx): Promise<Stats> {30  const { db, nft } = ctx;31  const scanTop = await getSetting<number>(db, 'collections.scan_top', 60);32  const maxListed = await getSetting<number>(db, 'collections.max_listed', 30);33  const perRun = await getSetting<number>(db, 'collections.list_per_run', 6);34 35  // 1. catalogue36  let catalog = await getSetting<CatalogRow[]>(db, 'collections.catalog', []);37  const catalogAt = Number(await getSetting<string>(db, 'collections.catalog_at', '0'));38  if (!catalog.length || Date.now() - catalogAt > DAY) {39    const list = await nft.howrare.list();40    const prev = new Map(catalog.map((c) => [c.slug, c]));41    catalog = list42      .filter((c) => c.meSymbol && c.items >= MIN_COLLECTION_SIZE && c.items <= MAX_COLLECTION_SIZE && c.holders >= 50)43      .sort((a, b) => b.holders - a.holders)44      .slice(0, scanTop)45      .map((c) => ({ floor: '0', vol7d: '0', volAll: '0', listed: 0, statsAt: 0, ...prev.get(c.slug), ...c }));46    await setSetting(db, 'collections.catalog', catalog);47    await setSetting(db, 'collections.catalog_at', String(Date.now()));48    ctx.log(`catalogue: ${list.length} collections on HowRare, ${catalog.length} candidates`);49  }50 51  // collections that must stay fresh: listed, or paired with a coin52  const paired = new Set(53    (await db.selectDistinct({ c: coins.collection }).from(coins).where(isNotNull(coins.collection))).map((r) => r.c!),54  );55  const known = await db.select({ address: collections.address, howrare: collections.howrare, status: collections.status, checkedAt: collections.checkedAt }).from(collections);56  const keepSlugs = new Set(known.filter((k) => k.status === 'listed' || paired.has(k.address)).map((k) => k.howrare));57 58  // 2. market data, stalest first, a bounded number per run (Magic Eden is slow)59  const statsBudget = await getSetting<number>(db, 'collections.stats_per_run', ctx.env.fast ? 60 : 25);60  const stale = catalog.filter((c) => Date.now() - c.statsAt > HOUR).sort((a, b) => a.statsAt - b.statsAt).slice(0, statsBudget);61  let statsOk = 0;62  for (const c of stale) {63    let s: MeStats;64    try {65      s = await nft.me.stats(c.meSymbol!);66    } catch (e) {67      ctx.log(`${c.name}: Magic Eden stats failed: ${(e as Error).message}`);68      continue;69    }70    Object.assign(c, { floor: s.floorLamports.toString(), vol7d: s.volume7dLamports.toString(), volAll: s.volumeAllLamports.toString(), listed: s.listedCount, statsAt: Date.now() });71    statsOk++;72    // keep listed rows in step73    await db74      .update(collections)75      .set({ floorLamports: s.floorLamports, volume7dLamports: s.volume7dLamports, volumeAllLamports: s.volumeAllLamports, listedCount: s.listedCount, marketAt: new Date() })76      .where(eq(collections.howrare, c.slug));77  }78  if (statsOk) await setSetting(db, 'collections.catalog', catalog);79 80  // 3. listing: the top traded candidates + everything we must keep, oldest check first81  const traded = catalog.filter((c) => BigInt(c.floor) > 0n && BigInt(c.vol7d) > 0n).sort((a, b) => (BigInt(b.vol7d) > BigInt(a.vol7d) ? 1 : -1));82  const want = [...new Set([...traded.slice(0, maxListed).map((c) => c.slug), ...keepSlugs].filter((s): s is string => !!s))];83  const checkedAt = new Map(known.map((k) => [k.howrare, k.checkedAt?.getTime() ?? 0]));84  // collections that can't be listed (no verified key, empty) are retried weekly, not every run85  for (const c of catalog) if (c.skipAt && !checkedAt.has(c.slug)) checkedAt.set(c.slug, c.skipAt + 6 * DAY);86  const due = want.filter((s) => Date.now() - (checkedAt.get(s) ?? 0) > DAY).sort((a, b) => (checkedAt.get(a) ?? 0) - (checkedAt.get(b) ?? 0)).slice(0, perRun);87 88  let listed = 0;89  let rejected = 0;90  let skipped = 0;91  for (const slug of due) {92    const cat = catalog.find((c) => c.slug === slug);93    try {94      const r = await listCollection(ctx, slug, cat);95      if (r === 'listed') listed++;96      else if (r === 'rejected') rejected++;97      else if (cat) {98        skipped++;99        cat.skipAt = Date.now();100      }101    } catch (e) {102      ctx.log(`${cat?.name ?? slug}: listing failed: ${(e as Error).message}`);103    }104  }105  if (skipped) await setSetting(db, 'collections.catalog', catalog);106  return { candidates: catalog.length, stats: statsOk, traded: traded.length, due: due.length, listed, rejected, skipped };107}108 109/** Download, verify, rank and upsert one collection. */110async function listCollection(ctx: JobCtx, slug: string, cat: CatalogRow | undefined): Promise<'listed' | 'rejected' | 'skipped'> {111  const { db, nft } = ctx;112  const hr = await nft.howrare.collection(slug);113  if (hr.items.length === 0) return 'skipped';114 115  // the verified collection key, from a spread of on-chain metadata accounts116  const step = Math.max(1, Math.floor(hr.items.length / 12));117  const sample = hr.items.filter((_, i) => i % step === 0).slice(0, 12).map((i) => i.mint);118  const md = (await nft.chain.metadata(sample)).filter((m): m is NonNullable<typeof m> => !!m);119  const keys = new Map<string, number>();120  let verifiedCount = 0;121  let programmable = 0;122  let checked = md.length;123  for (const m of md) {124    if (m.collection?.verified) {125      verifiedCount++;126      keys.set(m.collection.key, (keys.get(m.collection.key) ?? 0) + 1);127    }128    if (m.tokenStandard === 4) programmable++;129  }130  let standard = md.length > 0 ? (programmable === md.length ? 'pnft' : programmable === 0 ? 'nft' : 'mixed') : 'nft';131  if (md.length === 0) {132    // Metaplex Core: no metadata account; the asset account itself names its collection133    const core = (await nft.chain.coreAssets(sample)).filter((a): a is NonNullable<typeof a> => !!a);134    checked = core.length;135    for (const a of core) {136      if (!a.collection) continue;137      verifiedCount++;138      keys.set(a.collection, (keys.get(a.collection) ?? 0) + 1);139    }140    if (core.length) standard = 'core';141  }142  const [address, hits] = [...keys.entries()].sort((a, b) => b[1] - a[1])[0] ?? [null, 0];143  if (!address || hits < Math.max(2, Math.ceil(checked * 0.8))) {144    ctx.log(`${hr.name}: no verified on-chain collection key (${verifiedCount}/${checked} samples verified); not listed`);145    return 'skipped';146  }147 148  // rarity over every NFT HowRare knows149  const ranked = rankRarity(hr.items.map((i) => ({ id: i.mint, attributes: i.attributes })));150  const byId = new Map(ranked.items.map((r) => [r.id, r]));151  const rarityHash = createHash('sha256')152    .update(ranked.items.map((r) => `${r.id}:${r.rank}:${r.tier}`).join('\n'))153    .digest('hex');154 155  const check = checkCollection({156    name: hr.name,157    verified: true, // on-chain verified key found above, and HowRare/Magic Eden list it158    floorLamports: BigInt(cat?.floor ?? '0'),159    volumeAllLamports: BigInt(cat?.volAll ?? '0'),160    standard: true,161    size: hr.items.length,162    lastMintAt: null,163  });164 165  const [existing] = await db.select({ status: collections.status, slug: collections.slug }).from(collections).where(eq(collections.address, address)).limit(1);166  let siteSlug = existing?.slug ?? slugify(hr.name);167  if (!existing) {168    const clash = await db.select({ a: collections.address }).from(collections).where(eq(collections.slug, siteSlug)).limit(1);169    if (clash.length) siteSlug = `${siteSlug}-${address.slice(0, 4).toLowerCase()}`;170  }171  const status = check.ok ? 'listed' : existing?.status === 'listed' ? 'disabled' : 'rejected';172 173  await db.transaction(async (t) => {174    const values = {175      address,176      slug: siteSlug,177      name: hr.name.slice(0, 64),178      meSymbol: cat?.meSymbol ?? null,179      howrare: slug,180      image: hr.logo ?? cat?.logo ?? null,181      description: hr.description,182      twitter: hr.twitter,183      website: hr.website,184      standard,185      size: hr.items.length,186      owners: cat?.holders ?? 0,187      floorLamports: BigInt(cat?.floor ?? '0'),188      listedCount: cat?.listed ?? 0,189      volume7dLamports: BigInt(cat?.vol7d ?? '0'),190      volumeAllLamports: BigInt(cat?.volAll ?? '0'),191      verified: true,192      status: status as 'listed' | 'rejected' | 'disabled',193      failedChecks: check.failed,194      flat: ranked.flat,195      totalShares: ranked.totalWeight,196      traitTypes: ranked.traitTypes,197      rarityHash,198      checkedAt: new Date(),199      marketAt: cat?.statsAt ? new Date(cat.statsAt) : null,200    };201    await t202      .insert(collections)203      .values(values)204      .onConflictDoUpdate({ target: collections.address, set: { ...values, slug: sql`${collections.slug}`, address: sql`${collections.address}` } });205    // NFTs in chunks; owner fields are the owners job's, never touched here206    const rows = hr.items.map((i) => {207      const r = byId.get(i.mint)!;208      return { id: i.mint, collection: address, name: i.name, image: i.image, rank: r.rank, tier: r.tier, multiplier: r.multiplier, score: r.score, attributes: i.attributes };209    });210    for (let k = 0; k < rows.length; k += 1000) {211      await t212        .insert(nfts)213        .values(rows.slice(k, k + 1000))214        .onConflictDoUpdate({215          target: nfts.id,216          set: {217            name: sql`excluded.name`,218            // the owners job may have stored the on-chain (DAS) image: keep it over the HowRare mirror219            image: sql`COALESCE(${nfts.image}, excluded.image)`,220            rank: sql`excluded.rank`,221            tier: sql`excluded.tier`,222            multiplier: sql`excluded.multiplier`,223            score: sql`excluded.score`,224            attributes: sql`excluded.attributes`,225            updatedAt: new Date(),226          },227        });228    }229    // live shares = shares of NFTs not known to be burnt230    await t.execute(sql`UPDATE collections SET total_shares = COALESCE((SELECT SUM(multiplier) FROM nfts WHERE collection = ${address} AND NOT burnt), 0) WHERE address = ${address}`);231  });232 233  if (!existing || existing.status !== status) {234    await event(db, {235      collection: address,236      action: status === 'listed' ? 'collection_listed' : 'collection_rejected',237      message:238        status === 'listed'239          ? `${hr.name} is listed: ${hr.items.length.toLocaleString('en-US')} NFTs ranked into rarity tiers`240          : `${hr.name} is not listed (${check.failed.join(', ')})`,241      level: status === 'disabled' ? 'alert' : 'info',242    });243  }244  ctx.log(`${hr.name}: ${status} · ${hr.items.length} NFTs · key ${address.slice(0, 6)}… · ${ranked.flat ? 'flat' : `${ranked.traitTypes.length} traits`}`);245  return status === 'listed' ? 'listed' : 'rejected';246}247 248/** Collections by status, for other jobs. */249export async function listedCollections(ctx: JobCtx): Promise<Array<{ address: string; name: string; meSymbol: string | null; size: number }>> {250  return ctx.db251    .select({ address: collections.address, name: collections.name, meSymbol: collections.meSymbol, size: collections.size })252    .from(collections)253    .where(and(inArray(collections.status, ['listed', 'disabled'])));254}