# CrawlConsole -- Replit (Express + Node.js) Install Guide

This guide installs server-side crawler tracking into a Replit project. Replit
apps run on **Express** via **Node.js**, so the integration lives in an Express
middleware (`src/middlewares/crawlconsole.ts`) -- not in client code, and not in
an environment variable prefixed with `VITE_`.

---

## 1. Add the two secrets in Replit

Open **Tools -> Secrets** (the padlock icon in the sidebar) and add:

| Name                          | Value                         |
| ----------------------------- | ----------------------------- |
| `CRAWLCONSOLE_PROJECT_KEY`    | your CrawlConsole project key |
| `CRAWLCONSOLE_TRACKER_KEY`    | your CrawlConsole tracker key |

Rules:
- **Do NOT prefix with `VITE_`**. These must be runtime secrets readable from
  the server process, not build-time values bundled into the client.
- Names must match exactly -- the tracker reads `process.env.CRAWLCONSOLE_PROJECT_KEY`
  and `process.env.CRAWLCONSOLE_TRACKER_KEY`.

---

## 2. Create `src/lib/crawlconsole-tracking.ts`

```ts
// src/lib/crawlconsole-tracking.ts

const CRAWLCONSOLE_ENDPOINT = "https://analytics.crawlconsole.com/v1/track";

function config() {
  return {
    projectKey: process.env.CRAWLCONSOLE_PROJECT_KEY || "",
    trackerKey: process.env.CRAWLCONSOLE_TRACKER_KEY || "",
  };
}

function clientIp(headers: Record<string, string | string[] | undefined>): string {
  const header = (name: string) => {
    const v = headers[name.toLowerCase()];
    return Array.isArray(v) ? v[0] : (v ?? "");
  };
  return (
    header("x-real-ip") ||
    header("x-forwarded-for").split(",")[0].trim() ||
    ""
  );
}

function skip(pathname: string): boolean {
  return (
    pathname.startsWith("/assets/") ||
    pathname.startsWith("/_build/") ||
    pathname === "/favicon.ico" ||
    pathname === "/robots.txt" ||
    pathname.endsWith(".css") ||
    pathname.endsWith(".js") ||
    pathname.endsWith(".map") ||
    pathname.endsWith(".png") ||
    pathname.endsWith(".jpg") ||
    pathname.endsWith(".jpeg") ||
    pathname.endsWith(".svg") ||
    pathname.endsWith(".webp") ||
    pathname.endsWith(".ico")
  );
}

export function trackCrawlerRequest(options: {
  pathname: string;
  search: string;
  origin: string;
  userAgent: string;
  xForwardedFor: string;
  statusCode: number;
}): void {
  const { projectKey, trackerKey } = config();

  if (!projectKey || !trackerKey) {
    console.warn(
      "CrawlConsole server tracking is missing CRAWLCONSOLE_PROJECT_KEY or CRAWLCONSOLE_TRACKER_KEY.",
    );
    return;
  }

  const { pathname, search, origin, userAgent, xForwardedFor, statusCode } = options;

  if (skip(pathname)) return;

  const path = `${pathname}${search}`;

  fetch(CRAWLCONSOLE_ENDPOINT, {
    method: "POST",
    headers: {
      authorization: `Bearer ${trackerKey}`,
      "content-type": "application/json",
    },
    body: JSON.stringify({
      project_key: projectKey,
      user_agent: userAgent,
      path,
      full_path: `${origin}${path}`,
      status_code: statusCode,
      client_ip: clientIp({ "x-forwarded-for": xForwardedFor }),
      client_ip_raw: xForwardedFor,
    }),
  })
    .then(() => undefined)
    .catch(() => undefined);
}
```

---

## 3. Create `src/middlewares/crawlconsole.ts`

This is the Express equivalent of the Cloudflare Worker `fetch` handler. It fires
tracking **after** the response is sent so it never adds latency to your users.

```ts
// src/middlewares/crawlconsole.ts
import type { Request, Response, NextFunction } from "express";
import { trackCrawlerRequest } from "../lib/crawlconsole-tracking.js";

export function crawlconsoleMiddleware(
  req: Request,
  res: Response,
  next: NextFunction,
): void {
  res.on("finish", () => {
    const protocol = req.headers["x-forwarded-proto"] ?? "https";
    const host = req.headers.host ?? req.hostname;
    const origin = `${protocol}://${host}`;

    let pathname: string;
    let search: string;
    try {
      const url = new URL(req.url, origin);
      pathname = url.pathname;
      search = url.search;
    } catch {
      return;
    }

    trackCrawlerRequest({
      pathname,
      search,
      origin,
      userAgent: req.headers["user-agent"] ?? "",
      xForwardedFor: (req.headers["x-forwarded-for"] as string) ?? "",
      statusCode: res.statusCode,
    });
  });

  next();
}
```

---

## 4. Register the middleware in `src/app.ts`

Add the import and register the middleware **before** your routes:

```ts
import { crawlconsoleMiddleware } from "./middlewares/crawlconsole";

// ... existing middleware (cors, pino-http, etc.) ...

app.use(crawlconsoleMiddleware);

app.use("/api", router);
```