Initial commit of Websearch

websearch.ts

@@ -0,0 +1,82 @@
+import { Ollama } from "npm:ollama";
+import * as cheerio from "npm:cheerio@1.0.0";
+import { Readability } from "jsr:@paoramen/cheer-reader";
+import type { Config } from "./main.ts";
+
+export async function getNewsUrls(
+  config: Config,
+  query: string,
+): Promise<string[] | undefined> {
+  try {
+    const response = await fetch(
+      `${config.search_url}?q=${query}&format=json`,
+    );
+    if (!response.ok) {
+      throw new Error(
+        `Failed to fetch results for query "${query}": ${response.statusText}`,
+      );
+    }
+
+    const data = await response.json();
+    return data.results
+      .map((result: { url: string }) => result.url)
+      .slice(0, 3);
+  } catch (error: any) {
+    console.error(
+      `Error fetching news URLs for query "${query}":`,
+      error.message,
+    );
+    return undefined;
+  }
+}
+
+export async function cleanTextFromHtml(url: string): Promise<string> {
+  try {
+    const response = await fetch(url);
+    if (!response.ok) {
+      // ToDo: It would be great to fetch additional sources, or skip to next
+      throw new Error(`Failed to fetch ${url}: ${response.statusText}`);
+    }
+
+    const html = await response.text();
+    return htmlToText(html).trim();
+  } catch (error: any) {
+    console.error(`Error fetching URL '${url}':`, error.message);
+    throw error;
+  }
+}
+
+function htmlToText(html: string): string {
+  const $ = cheerio.load(html);
+  return new Readability($).parse().textContent || "";
+}
+
+export async function answerQuery(
+  config: Config,
+  query: string,
+  texts: string,
+) {
+  const ollama = new Ollama();
+  if (!config.model) {
+    throw new Error(`No model in config: ${config}`);
+  }
+  try {
+    const responseStream = await ollama.generate({
+      model: config.model,
+      prompt:
+        `For the topic of ${query}, provide a summary of the information in the following articles:\n${texts}`,
+      stream: true,
+    });
+
+    for await (const chunk of responseStream) {
+      if (!chunk.done) {
+        await Deno.stdout.write(new TextEncoder().encode(chunk.response));
+      }
+    }
+  } catch (error: any) {
+    console.error("Error answering query:", error.message);
+    throw error;
+  } finally {
+    void ollama;
+  }
+}