Product Hunt Catalog

{  "type": "object",  "additionalProperties": false,  "properties": {    "mode": {      "type": "string",      "enum": [        "backfill",        "discover",        "sync",        "drain"      ],      "title": "Mode",      "description": "backfill — walk every category listing into the product queue, then scrape the queue. discover — walk each category newest-first until the pages hold only known products, then scrape what is new. sync — rescrape products older than syncAfterDays, oldest first. drain — only scrape what is already queued. Every mode scrapes new products before stale ones.",      "default": "backfill"    },    "syncAfterDays": {      "type": "number",      "minimum": 1,      "maximum": 3650,      "title": "Resync after (days)",      "description": "In sync mode, products last scraped longer ago than this are rescraped.",      "default": 30    },    "maxDiscoverPagesPerCategory": {      "type": "number",      "minimum": 1,      "maximum": 10000,      "title": "Max newest-first pages per category",      "description": "Discover mode stops a category after two pages with no unknown product, or after this many pages.",      "default": 50    },    "followLinks": {      "type": "boolean",      "title": "Queue linked products",      "description": "Queue products seen in alternatives, customers and makers' product lists. Catches products filed under no category, which no listing ever shows.",      "default": true    },    "categoriesInParallel": {      "type": "number",      "minimum": 1,      "maximum": 64,      "title": "Category listings in parallel",      "default": 8    },    "cookie": {      "type": "string",      "maxLength": 2000,      "title": "Cloudflare cookie",      "description": "Usually unnecessary. Cloudflare only challenges some IPs (typically VPN or datacenter ranges); from those, solve the challenge in a browser on the same IP and paste its cf_clearance (the bare value or a full Cookie header) with that browser's userAgent.",      "examples": [        "cf_clearance=…"      ]    },    "userAgent": {      "type": "string",      "maxLength": 2000,      "title": "User-Agent",      "description": "Only matters with a cookie: use the User-Agent of the browser that produced it.",      "default": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:156.0) Gecko/20100101 Firefox/156.0"    },    "impersonation": {      "type": "string",      "enum": [        "firefox",        "chrome"      ],      "title": "TLS profile",      "description": "Match the browser family of the User-Agent.",      "default": "firefox"    },    "useProxy": {      "type": "boolean",      "title": "Use the proxy pool",      "description": "Off by default: a rotating pool mixes IPs Cloudflare lets through with ones it challenges.",      "default": false    },    "categories": {      "type": "array",      "items": {        "type": "string"      },      "maxItems": 1000,      "title": "Category slugs",      "description": "Empty discovers every category from /categories, recursing into subcategories.",      "examples": [        "engineering-development"      ]    },    "products": {      "type": "array",      "items": {        "type": "string"      },      "maxItems": 1000,      "title": "Product slugs",      "description": "Scrape exactly these products and skip category discovery.",      "examples": [        "vercel"      ]    },    "includeSubcategories": {      "type": "boolean",      "title": "Include subcategories",      "default": true    },    "featuredOnly": {      "type": "boolean",      "title": "Featured products only",      "description": "The site defaults to featured products; off collects the full category listing.",      "default": false    },    "maxProducts": {      "type": "number",      "minimum": 1,      "maximum": 1000000,      "title": "Max products per run",      "default": 1000000    },    "maxCategories": {      "type": "number",      "minimum": 1,      "maximum": 5000,      "title": "Max categories per run",      "default": 5000    },    "maxPagesPerCategory": {      "type": "number",      "minimum": 1,      "maximum": 10000,      "title": "Max listing pages per category",      "description": "Twenty products per page; Product Hunt caps the page size.",      "default": 10000    },    "refreshAfterHours": {      "type": "number",      "minimum": 0,      "maximum": 8760,      "title": "Refresh after (hours)",      "description": "How long a category listing checkpoint stays valid for backfill resumes, and, with rescrapeKnownMakers, how old a maker must be to be refetched.",      "default": 24    },    "resumeCategories": {      "type": "boolean",      "title": "Resume category listings",      "description": "Checkpoints each category listing in lead_gen_product_hunt_crawl_state. Within refreshAfterHours a rerun skips categories already walked to the end and restarts partial ones one page before where they stopped, instead of re-reading every listing from page 1.",      "default": true    },    "productSections": {      "type": "array",      "items": {        "type": "string"      },      "maxItems": 1000,      "title": "Product sections",      "description": "Empty fetches all of: layout, about, reviewSummary, reviews, launches, alternatives, customers, team, awards, embed.",      "examples": [        "layout, about, reviewSummary, reviews, launches, alternatives, customers, team, awards, embed"      ]    },    "scrapeTeamProfiles": {      "type": "boolean",      "title": "Scrape team member profiles",      "description": "Writes each maker to lead_gen_product_hunt_makers.",      "default": true    },    "rescrapeKnownMakers": {      "type": "boolean",      "title": "Rescrape known makers",      "description": "Off: a maker already in lead_gen_product_hunt_makers is never fetched again, only linked to the new product. On: makers older than refreshAfterHours are fetched and refreshed in place.",      "default": false    },    "profileSections": {      "type": "array",      "items": {        "type": "string"      },      "maxItems": 1000,      "title": "Profile sections",      "description": "Empty fetches all of: layout, about, activity, upvotes, forums, collections, stacks, reviews.",      "examples": [        "layout, about, activity, upvotes, forums, collections, stacks, reviews"      ]    },    "maxReviewsPerProduct": {      "type": "number",      "minimum": 0,      "maximum": 20000,      "title": "Max reviews per product",      "default": 1000    },    "maxLaunchesPerProduct": {      "type": "number",      "minimum": 0,      "maximum": 5000,      "title": "Max launches per product",      "default": 500    },    "maxAlternativesPerProduct": {      "type": "number",      "minimum": 0,      "maximum": 2000,      "title": "Max alternatives per product",      "default": 100    },    "maxCustomersPerProduct": {      "type": "number",      "minimum": 0,      "maximum": 5000,      "title": "Max customers per product",      "default": 500    },    "maxTeamPerProduct": {      "type": "number",      "minimum": 0,      "maximum": 5000,      "title": "Max team members per product",      "default": 500    },    "maxAwardsPerProduct": {      "type": "number",      "minimum": 0,      "maximum": 5000,      "title": "Max awards per product",      "default": 500    },    "concurrency": {      "type": "number",      "minimum": 1,      "maximum": 128,      "title": "Products in parallel",      "description": "Products scraped at once. Each one fetches its sections in parallel and its team four profiles at a time, all under maxInFlight.",      "default": 32    },    "maxInFlight": {      "type": "number",      "minimum": 1,      "maximum": 256,      "title": "Max requests in flight",      "description": "Global cap on simultaneous requests to Product Hunt, whatever the concurrency. A 429 pauses every worker together.",      "default": 96    },    "requestDelayMs": {      "type": "number",      "minimum": 0,      "maximum": 60000,      "title": "Delay between requests (ms)",      "default": 0    },    "maxConsecutiveBlocks": {      "type": "number",      "minimum": 1,      "maximum": 100,      "title": "Abort after consecutive blocks",      "description": "A challenged request is retried after a pause; once Cloudflare refuses this many requests in a row the run stops and keeps what it collected. High enough to ride out an isolated challenge under heavy concurrency, low enough that a challenged IP stops within seconds.",      "default": 10    },    "maxOfflineMinutes": {      "type": "number",      "minimum": 1,      "maximum": 1440,      "title": "Abort after offline (minutes)",      "description": "Connection failures pause the crawl and retry with backoff instead of skipping work. If Product Hunt stays unreachable this long, the run stops cleanly so it can be resumed.",      "default": 30    }  }}