Read reference

type ReadResult = ReadReport & { readonly content: readonly ContentPart[] };
type ReadReport = ReadOk | ReadMedia | ReadUnsupported | ReadFailure;

type ContentPart =
  | { type: "text"; text: string }
  | { type: "media"; mediaType: string; data: Uint8Array; name?: string };

import { memoryFileSystem } from "@better-fs-tools/fs";
import { createReadTool, textOf } from "@better-fs-tools/read";

const read = createReadTool({
  fs: memoryFileSystem({ files: { "/notes.txt": "one\ntwo\nthree\n" } }),
});

const result = await read({ path: "/notes.txt", limit: 2 });

if (result.status === "ok") {
  console.log(result.view.lines.map((line) => line.text)); // ["one", "two"]
  console.log(result.truncation.primary); // "lines"
  console.log(result.continuation.next); // { path: "/notes.txt", offset: 3, limit: 2 }
} else if (result.status === "error") {
  console.log(result.error.code, result.error.phase);
}
console.log(result.notes.map((note) => note.code)); // ["continue"]
console.log(textOf(result));

interface ReadOk {
  tool: "read";
  status: "ok";
  request: { path: string; offset: number; limit: number; ranged: boolean };
  file: FileInfo;
  classification: {
    kind: "text" | "unsupported" | "directory";
    classifier: string;
    code: string | null;
    mimeType: string | null;
    confidence: "high" | "medium" | "low";
    reasons: readonly string[];
  };
  conversion: { converter: string; mimeType: string | null } | null;
  view: {
    lines: readonly {
      number: number;
      text: string;
      clamped: boolean;
      sourceChars: number | null;
    }[];
    startLine: number;
    endLine: number;
    bytes: number;
    partial: boolean;
  };
  truncation: {
    truncated: boolean;
    reasons: readonly TruncationReason[];
    primary: TruncationReason | null;
  };
  continuation: { available: boolean; next: ReadInput | null };
  totals: { lines: number | null; exact: boolean; bytes: number };
  observation: ReadObservation | null;
  notes: readonly ReadNote[];
}

interface FileInfo {
  requestedPath: string;
  resolvedPath: string;
  displayPath: string;
  backend: string;
  size: number | null;
  mtimeMs: number | null;
  identity: string | null;
  mimeType: string | null;
  resolvedFrom: string | null;
  version: string | null;
}

interface ReadObservation {
  id: string;
  statId: string;
  contentId: string | null;
  viewId: string;
  observedAt: string;
  wholeFileVisible: boolean;
  visibleLines: { start: number; end: number } | null;
}

type TruncationReason = "lines" | "bytes" | "budget" | "line-length" | "scan-limit";

interface ReadMedia {
  tool: "read";
  status: "media";
  request: ReadRequest;
  file: FileInfo;
  classification: ClassificationInfo;
  conversion: { converter: string; mimeType: string | null };
  parts: readonly ContentPart[];
  observation: ReadObservation | null; // wholeFileVisible is false
  notes: readonly ReadNote[];
}

interface ReadUnsupported {
  tool: "read";
  status: "unsupported";
  code: string;
  request: ReadRequest;
  file: FileInfo;
  classification: ClassificationInfo;
  notes: readonly ReadNote[];
}

interface ReadFailure {
  tool: "read";
  status: "error";
  error: { code: ReadErrorCode; phase: ReadPhase; message: string; data?: JsonObject };
  request: ReadRequest | null;
  file: FileInfo | null;
  notes: readonly ReadNote[];
}

interface ReadNote {
  code: string;
  severity: "info" | "warning";
  message: string;
  retry?: ReadInput;
  data?: JsonObject;
}

interface ReadStateStore {
  get(key: string): Promise<ReadRecord | null>;
  put(key: string, record: ReadRecord): Promise<void>;
  delete(key: string): Promise<void>;
}

interface ReadRecord {
  schema: 2;
  origin: "read" | "write";
  observationId: string;
  resolvedPath: string;
  identity: string | null;
  version: string | null; // file.version
  digest: string; // the Digest.id that made contentId and viewId
  contentId: string | null;
  viewId: string;
  observedAt: string;
  wholeFileVisible: boolean;
  seenLines: { start: number; end: number }[]; // empty for a write record
  totalsExact: boolean;
  request: { offset: number; limit: number } | null; // null for a write record
}

import { memoryFileSystem } from "@better-fs-tools/fs";
import { createReadTool, sha256Digest } from "@better-fs-tools/read";
import type { ReadRecord, ReadStateStore } from "@better-fs-tools/read";

// A store over a Map. Put your own database calls here.
const records = new Map<string, ReadRecord>();
const store: ReadStateStore = {
  get: async (key) => records.get(key) ?? null,
  put: async (key, record) => {
    records.set(key, record);
  },
  delete: async (key) => {
    records.delete(key);
  },
};

const read = createReadTool({
  fs: memoryFileSystem({ files: { "/a.txt": "alpha\n" } }),
  state: store,
  digest: sha256Digest(),
});

await read({ path: "/a.txt" });
const record = records.get("/a.txt");
console.log(record?.origin, record?.wholeFileVisible); // read true

interface FileConverter<THost> {
  readonly id: string;
  readonly target: "file";
  accepts(match: ConverterMatch): boolean; // synchronous. The only place to decline
  convert(input: FileConvertInput, ctx: ReadHookContext<THost>): Promise<ConvertOutcome>;
}

interface ConverterMatch {
  readonly classification: Classification;
  readonly classifier: string;
  readonly sample: ClassificationSample;
}

interface FileConvertInput {
  readonly info: Readonly<OpenFileInfo>;
  readonly classification: Classification;
  readonly sample: ClassificationSample;
  bytes(): AsyncIterable<Uint8Array>; // from byte 0, cut at limits.maxConvertBytes. Call it once
}

interface DirectoryConverter<THost> {
  readonly id: string;
  readonly target: "directory";
  convert(input: DirectoryConvertInput, ctx: ReadHookContext<THost>): Promise<ConvertOutcome>;
}

interface DirectoryConvertInput {
  readonly path: string; // the path that open() reported as a folder
  readonly target: { readonly resolvedPath: string; readonly displayPath: string } | null;
  list(): Promise<ListOutcome>; // authorizes and lists, up to maxDirectoryEntries. Call it once
}

type ConvertOutcome =
  | {
      kind: "text";
      text: string | AsyncIterable<string>;
      mimeType: string | null;
      notes?: readonly ReadNote[];
    }
  | { kind: "media"; parts: readonly ContentPart[]; notes?: readonly ReadNote[] }
  | { kind: "refuse"; code: string; note: ReadNote };

import { memoryFileSystem } from "@better-fs-tools/fs";
import { createReadTool, defaultClassifiers, textOf } from "@better-fs-tools/read";
import type { FileConverter } from "@better-fs-tools/read";

// Reads a PDF as a fixed line of text. Put your own extraction here.
const pdfStub: FileConverter<unknown> = {
  id: "pdf-stub",
  target: "file",
  accepts: (match) =>
    match.classification.kind === "unsupported" && match.classification.code === "PDF",
  convert: async (input) => {
    let bytes = 0;
    for await (const chunk of input.bytes()) bytes += chunk.byteLength;
    return { kind: "text", text: `PDF with ${bytes} bytes\n`, mimeType: "text/plain" };
  },
};

const read = createReadTool({
  fs: memoryFileSystem({ files: { "/a.pdf": "%PDF-1.7\n" } }),
  classifiers: defaultClassifiers(),
  converters: [pdfStub],
});

const result = await read({ path: "/a.pdf" });
console.log(result.status); // "ok"
console.log(textOf(result)); // "1|PDF with 9 bytes"

interface Classifier {
  readonly id: string;
  classify(sample: ClassificationSample): Classification | null;
  // A refusal for bad UTF-8 that the tool finds after the sample was accepted as text.
  encodingFailure?(sample: ClassificationSample): UnsupportedClassification | null;
}

interface ClassificationSample {
  readonly bytes: Uint8Array; // at most limits.sampleBytes
  readonly complete: boolean; // true when the sample is the whole file
  readonly path: string; // the display path
  readonly mimeType: string | null; // from the backend
}

import {
  binaryClassifier,
  canonicalizeFileName,
  compileGlob,
  defaultNoteLine,
  parseReadInput,
  resolveReadLimits,
  resolveReadMessages,
  suggestFileNames,
  utf8Classifier,
} from "@better-fs-tools/read";

const limits = resolveReadLimits({ maxLines: 100 });
console.log(parseReadInput({ path: "src/a.ts", limit: 500 }, limits));
// { path: "src/a.ts", offset: 1, limit: 100, ranged: true }

try {
  parseReadInput({ file_path: "src/a.ts" }, limits);
} catch (error) {
  console.log((error as TypeError).message); // "Unknown read input key: file_path"
}

const messages = resolveReadMessages({ empty: ({ path }) => `${path} has no content.` });
console.log(messages.empty({ path: "a.txt" })); // "a.txt has no content."

const isEnv = compileGlob("**/.env");
console.log(isEnv("/repo/.env"), isEnv("/repo/.env.local")); // true false

console.log(suggestFileNames("READMe.md", ["README.md", "src", "package.json"], 5)); // ["README.md"]
console.log(canonicalizeFileName("Screen Shot 10.42 AM.png")); // "Screen Shot 10.42 AM.png"

console.log(defaultNoteLine({ code: "empty", severity: "info", message: "a.txt is empty." }));
// "[read:empty] a.txt is empty."

const sample = (text: string) => ({
  bytes: new TextEncoder().encode(text),
  complete: true,
  path: "/a.txt",
  mimeType: null,
});
console.log(utf8Classifier().classify(sample("hello\n"))?.kind); // "text"
console.log(binaryClassifier().classify(sample("a\0b"))?.kind); // "unsupported"
console.log(binaryClassifier().classify(sample("hello\n"))); // null: no opinion