apps/keeper/src/nft/sources.ts

1// Where real NFT data comes from. All read-only.2//   HowRare.is   : the collection catalogue, every NFT's name, image and traits (public, no key)3//   Magic Eden   : floor, listings, volume; per-token owner as a slow fallback (public, rate-limited)4//   Helius DAS   : every NFT's current owner, burns, compression, verified grouping (needs HELIUS_API_KEY)5//   Solana RPC   : Metaplex metadata accounts (the verified collection key), wallet existence6import { decodeCoreAsset, decodeMetadata, MPL_CORE_PROGRAM, TOKEN_METADATA_PROGRAM, type CoreAsset, type MetadataAccount } from '@solary/core/metaplex';7import { findPda, isOnCurve, pk } from '../solana/keys';8import { SolRpc } from '../solana/rpc';9import { sleep } from '../ctx';10 11const UA = { 'user-agent': 'solary-keeper/0.1 (+https://solary.fun)', accept: 'application/json' };12 13/** One request at a time with a minimum gap; waits and retries on rate limits. */14class Throttle {15  private tail: Promise<unknown> = Promise.resolve();16  private last = 0;17  constructor(private readonly gapMs: number) {}18  run<T>(fn: () => Promise<T>): Promise<T> {19    const next = this.tail.then(async () => {20      const wait = this.last + this.gapMs - Date.now();21      if (wait > 0) await sleep(wait);22      try {23        return await fn();24      } finally {25        this.last = Date.now();26      }27    });28    this.tail = next.catch(() => undefined);29    return next;30  }31}32 33async function getJson<T>(url: string, throttle: Throttle, opts: { retries?: number; timeoutMs?: number } = {}): Promise<T> {34  const retries = opts.retries ?? 3;35  for (let attempt = 0; ; attempt++) {36    const res = await throttle.run(async () => {37      const r = await fetch(url, { headers: UA, signal: AbortSignal.timeout(opts.timeoutMs ?? 30_000) });38      const text = await r.text();39      return { status: r.status, text };40    });41    const limited = res.status === 429 || /exceeded the requests/i.test(res.text);42    if (res.status === 200 && !limited) {43      try {44        return JSON.parse(res.text) as T;45      } catch {46        throw new Error(`${new URL(url).host}: bad JSON`);47      }48    }49    if (res.status === 404) throw new NotFound(url);50    if (attempt >= retries) throw new Error(`${new URL(url).host}: HTTP ${res.status}${limited ? ' (rate limited)' : ''}`);51    await sleep(limited ? 20_000 * (attempt + 1) : 2_000 * (attempt + 1));52  }53}54 55export class NotFound extends Error {56  constructor(url: string) {57    super(`not found: ${url}`);58  }59}60 61// ---------------------------------------------------------------- HowRare62 63export interface HowRareListing {64  slug: string;65  name: string;66  logo: string | null;67  meSymbol: string | null;68  holders: number;69  items: number;70}71 72export interface HowRareItem {73  mint: string;74  name: string;75  image: string | null;76  attributes: Array<{ trait_type: string; value: string }>;77}78 79export interface HowRareCollection {80  name: string;81  description: string;82  logo: string | null;83  twitter: string | null;84  website: string | null;85  items: HowRareItem[];86}87 88const cleanUrl = (u: unknown) => (typeof u === 'string' && /^https?:\/\//.test(u) ? u.replace(/^http:/, 'https:').slice(0, 300) : null);89 90export class HowRare {91  private readonly t = new Throttle(1_500);92  constructor(private readonly base = 'https://api.howrare.is/v0.1') {}93 94  async list(): Promise<HowRareListing[]> {95    type Row = { name: string; url: string; logo?: string; me_key?: string; holders?: number; items?: number };96    const j = await getJson<{ result: { data: Row[] } }>(`${this.base}/collections`, this.t);97    return j.result.data98      .filter((r) => typeof r.url === 'string' && r.url.startsWith('/'))99      .map((r) => ({100        slug: r.url.slice(1),101        name: r.name,102        logo: cleanUrl(r.logo),103        meSymbol: r.me_key || null,104        holders: Number(r.holders ?? 0),105        items: Number(r.items ?? 0),106      }));107  }108 109  async collection(slug: string): Promise<HowRareCollection> {110    type Item = { mint: string; name: string; image?: string; attributes?: Array<{ name: string; value: string }> };111    type Data = { collection: string; description?: string; logo?: string; twitter?: string; website?: string; items: Item[] };112    const j = await getJson<{ result: { data: Data } }>(`${this.base}/collections/${encodeURIComponent(slug)}`, this.t, { timeoutMs: 90_000 });113    const d = j.result.data;114    return {115      name: d.collection,116      description: (d.description ?? '').slice(0, 1000),117      logo: cleanUrl(d.logo),118      twitter: cleanUrl(d.twitter),119      website: cleanUrl(d.website),120      items: (d.items ?? [])121        .filter((i) => typeof i.mint === 'string')122        .map((i) => ({123          mint: i.mint,124          name: String(i.name ?? '').slice(0, 64),125          image: cleanUrl(i.image),126          // HowRare adds its own "Attribute count"; the rarity code counts traits itself127          attributes: (i.attributes ?? [])128            .filter((a) => a && a.name && a.name.toLowerCase() !== 'attribute count')129            .map((a) => ({ trait_type: String(a.name), value: String(a.value) })),130        })),131    };132  }133}134 135// ---------------------------------------------------------------- Magic Eden136 137export interface MeStats {138  floorLamports: bigint;139  listedCount: number;140  volume7dLamports: bigint;141  volumeAllLamports: bigint;142  avgPrice24hLamports: bigint;143}144 145const lamports = (v: unknown) => (typeof v === 'number' && Number.isFinite(v) && v > 0 ? BigInt(Math.round(v)) : 0n);146 147export class MagicEden {148  // the public API allows a couple of requests per second at best; stay well under it149  private readonly t = new Throttle(1_300);150  constructor(private readonly base = 'https://api-mainnet.magiceden.dev/v2') {}151 152  async stats(symbol: string): Promise<MeStats> {153    const j = await getJson<Record<string, unknown>>(`${this.base}/collections/${encodeURIComponent(symbol)}/stats`, this.t);154    return {155      floorLamports: lamports(j.floorPrice),156      listedCount: Number(j.listedCount ?? 0) || 0,157      volume7dLamports: lamports(j.volume7d),158      // Magic Eden's stats have no all-time volume (2026-09): 0 = unknown, never the 7-day figure159      volumeAllLamports: lamports(j.volumeAll),160      avgPrice24hLamports: lamports(j.avgPrice24hr),161    };162  }163 164  /** The current owner of one NFT (slow: one request per NFT). */165  async owner(mint: string): Promise<{ owner: string | null; listed: boolean }> {166    const j = await getJson<{ owner?: string; listStatus?: string }>(`${this.base}/tokens/${mint}`, this.t, { retries: 2 });167    return { owner: typeof j.owner === 'string' ? j.owner : null, listed: j.listStatus === 'listed' };168  }169}170 171// ---------------------------------------------------------------- Helius DAS172 173export interface DasAsset {174  id: string;175  owner: string | null;176  burnt: boolean;177  compressed: boolean;178  /** V1_NFT | ProgrammableNFT | MplCoreAsset | ... */179  interface: string;180  name: string;181  image: string | null;182  attributes: Array<{ trait_type: string; value: string }>;183}184 185export class HeliusDas {186  private readonly t = new Throttle(120);187  constructor(private readonly apiKey: string) {}188 189  private async call<T>(method: string, params: unknown): Promise<T> {190    for (let attempt = 0; ; attempt++) {191      const r = await this.t.run(() =>192        fetch(`https://mainnet.helius-rpc.com/?api-key=${this.apiKey}`, {193          method: 'POST',194          headers: { 'content-type': 'application/json' },195          body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }),196          signal: AbortSignal.timeout(60_000),197        }),198      );199      if (r.status === 429 && attempt < 4) {200        await sleep(2_000 * (attempt + 1));201        continue;202      }203      if (!r.ok) throw new Error(`helius ${method}: HTTP ${r.status}`); // never print the URL: it holds the key204      const j = (await r.json()) as { result?: T; error?: { message: string } };205      if (j.error) throw new Error(`helius ${method}: ${j.error.message}`);206      return j.result as T;207    }208  }209 210  /** Every asset in a verified collection, 1 000 per page. */211  async collectionAssets(collection: string, onPage?: (n: number) => void): Promise<DasAsset[]> {212    type Raw = {213      id: string;214      interface: string;215      burnt?: boolean;216      ownership?: { owner?: string };217      compression?: { compressed?: boolean };218      content?: { metadata?: { name?: string; attributes?: Array<{ trait_type?: string; value?: unknown }> }; links?: { image?: string } };219    };220    const out: DasAsset[] = [];221    for (let page = 1; page <= 50; page++) {222      const r = await this.call<{ items: Raw[]; total: number }>('getAssetsByGroup', { groupKey: 'collection', groupValue: collection, page, limit: 1000 });223      for (const a of r.items) {224        out.push({225          id: a.id,226          owner: a.ownership?.owner ?? null,227          burnt: !!a.burnt,228          compressed: !!a.compression?.compressed,229          interface: a.interface,230          name: String(a.content?.metadata?.name ?? '').slice(0, 64),231          image: cleanUrl(a.content?.links?.image),232          // some metadata carries attributes as an object, not a list233          attributes: (Array.isArray(a.content?.metadata?.attributes) ? a.content!.metadata!.attributes! : [])234            .filter((x) => x && typeof x.trait_type === 'string')235            .map((x) => ({ trait_type: String(x.trait_type), value: String(x.value) })),236        });237      }238      onPage?.(out.length);239      if (r.items.length < 1000) break;240    }241    return out;242  }243}244 245// ---------------------------------------------------------------- chain reads246 247export function metadataPda(mint: string): string {248  return findPda(['metadata', pk(TOKEN_METADATA_PROGRAM), pk(mint)], TOKEN_METADATA_PROGRAM);249}250 251/** A wallet (on the ed25519 curve) rather than a program address (escrow, pool, PDA). */252export function isWalletAddress(address: string): boolean {253  try {254    return isOnCurve(pk(address));255  } catch {256    return false;257  }258}259 260export class ChainReads {261  private readonly rpc: SolRpc;262  private readonly t = new Throttle(250);263  constructor(rpcUrl: string) {264    this.rpc = new SolRpc(rpcUrl);265  }266 267  /** Metaplex metadata for up to 100 mints (null where missing or unreadable). */268  async metadata(mints: readonly string[]): Promise<Array<MetadataAccount | null>> {269    const out: Array<MetadataAccount | null> = [];270    for (let i = 0; i < mints.length; i += 100) {271      const chunk = mints.slice(i, i + 100).map(metadataPda);272      const accts = await this.t.run(() => this.rpc.getAccounts(chunk));273      out.push(...accts.map((a) => (a ? decodeMetadata(a.data) : null)));274    }275    return out;276  }277 278  /** Metaplex Core assets (owner + collection straight from the asset account), null where not a Core asset. */279  async coreAssets(ids: readonly string[]): Promise<Array<CoreAsset | null>> {280    const out: Array<CoreAsset | null> = [];281    for (let i = 0; i < ids.length; i += 100) {282      const accts = await this.t.run(() => this.rpc.getAccounts(ids.slice(i, i + 100)));283      out.push(...accts.map((a) => (a && a.owner === MPL_CORE_PROGRAM ? decodeCoreAsset(a.data) : null)));284    }285    return out;286  }287 288  /** Which of these addresses exist on-chain (hold lamports). */289  async existing(addresses: readonly string[]): Promise<Set<string>> {290    const found = new Set<string>();291    for (let i = 0; i < addresses.length; i += 100) {292      const chunk = addresses.slice(i, i + 100);293      const accts = await this.t.run(() => this.rpc.getAccounts(chunk));294      accts.forEach((a, j) => {295        if (a && a.lamports > 0) found.add(chunk[j]!);296      });297    }298    return found;299  }300}301 302export interface NftSources {303  howrare: HowRare;304  me: MagicEden;305  das: HeliusDas | null;306  chain: ChainReads;307}308 309export function nftSources(env: { heliusKey?: string; publicRpcUrl: string }): NftSources {310  return { howrare: new HowRare(), me: new MagicEden(), das: env.heliusKey ? new HeliusDas(env.heliusKey) : null, chain: new ChainReads(env.publicRpcUrl) };311}