Code changes for running LLM

Author: arnab2001

.gitignore

@@ -66,4 +66,6 @@ yarn-error.log*
 
 # typescript
 *.tsbuildinfo
-next-env.d.ts
+next-env.d.ts
+
+.env

package.json

@@ -6,6 +6,7 @@
     "dev": "next dev",
     "build": "next build",
     "start": "next start",
+    "prepare:data": "tsx -r dotenv/config ./src/scripts/pinecone-prepare-docs.ts",
     "lint": "next lint --quiet",
     "lint:fix": "next lint --fix",
     "prepare": "husky install",
@@ -27,6 +28,7 @@
     "ai-stream-experimental": "^2.2.2",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
+    "dotenv": "^16.4.5",
     "framer-motion": "^11.2.12",
     "git-repo-parser": "^2.0.6",
     "langchain": "^0.2.8",
@@ -59,6 +61,7 @@
     "prettier": "^3.2.5",
     "prettier-plugin-tailwindcss": "^0.5.11",
     "tailwindcss": "^3.4.1",
+    "tsx": "^4.16.2",
     "typescript": "^5.3.3"
   }
 }

src/scripts/pinecone-prepare-docs.ts

@@ -43,7 +43,7 @@ import { embedAndStoreDocs } from "@/lib/vector-store";
         };
       });
 
-    const validDocs = [...docs, ...directories];
+    const validDocs = [...docs];
 
     if (validDocs.length === 0) {
       throw new Error("No valid documents to process.");