250kb-club/analyser/toolkit.ts

import {
  parse as tomlParse,
  stringify as tomlStringify,
} from "https://deno.land/std@0.130.0/encoding/toml.ts";

const reFrontmatter = /^\+\+\+([\s\S]*)^\+\+\+$([\s\S]*)/m;

export function url2title(url: string): string {
  return url
    .replace(/^https?:\/\//, "") // remove leading http(s)://
    .replace(/\/$/, ""); // remove trailing slash
}

// gets an URL like https://foo.bar and returns ./content/foo_baz.md
function url2filepath(url: string, output_path: string): string {
  const filename = url2title(url).replaceAll(/[\.\/]/g, "_"); // replace dots and slashes with underscores
  return `${output_path}/${filename}.md`;
}

// deprecated in deno std, but also simple to replicate
// see: https://deno.land/std@0.130.0/fs/exists.ts
async function exists(path: string): Promise<boolean> {
  try {
    return !!(await Deno.lstat(path));
  } catch (err) {
    if (err instanceof Deno.errors.NotFound) return false;
    throw err;
  }
}

// checks if URL has a record already and returns time since last check or null
export async function getPageRecord(
  url: string,
  output_path: string
): Promise<PageRecord | null> {
  const path = url2filepath(url, output_path);
  const hasRecord = await exists(path);

  if (!hasRecord) return null;

  const fileContents = await Deno.readTextFile(path);
  const match = fileContents.match(reFrontmatter);
  if (!match) return null; // that should never happen but who knows
  return tomlParse(match[1].trim()) as PageRecord;
}

export async function writeRecord(
  record: PageRecord,
  url: string,
  output_path: string
): Promise<boolean> {
  const path = url2filepath(url, output_path);
  const toml = tomlStringify(record);

  try {
    await Deno.writeTextFile(path, `+++\n${toml}+++\n`);
    return true;
  } catch {
    return false;
  }
}

export async function removeRecord(url: string, output_path: string) {
  const path = url2filepath(url, output_path);
  const hasRecord = await exists(path);

  if (!hasRecord) return false;

  try {
    await Deno.remove(path);
    return true;
  } catch {
    return false;
  }
}

function delay(ms: number): Promise<unknown> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

export async function retryFetch(
  url: string,
  retries = 3,
  msDelay = 1000
): Promise<any> {
  try {
    const response = await fetch(url);
    if (!response.ok) return false;
    const json = await response.json();
    return json;
  } catch (err) {
    if (retries > 0) {
      console.warn(`Failed to fetch ${url}, retrying in ${msDelay}ms.`);
      await delay(msDelay);
      return retryFetch(url, retries - 1, msDelay);
    } else {
      console.error(`Fetching ${url} failed too often. Giving up.`);
      return false;
    }
  }
}