🐛 fix: improve url rules (#6669)

* improve code * improve code
lobehub · Mar 3, 2025 · 5ee59e3 · 5ee59e3
1 parent 1a0330c
commit 5ee59e3
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/packages/web-crawler/src/crawImpl/jina.ts b/packages/web-crawler/src/crawImpl/jina.ts
@@ -1,7 +1,7 @@
 import { CrawlImpl } from '../type';
 
 export const jina: CrawlImpl<{ apiKey?: string }> = async (url, params) => {
-  const token = params.apiKey ?? process.env.JINA_API_KEY;
+  const token = params.apiKey ?? process.env.JINA_READER_API_KEY ?? process.env.JINA_API_KEY;
 
   try {
     const res = await fetch(`https://r.jina.ai/${url}`, {

diff --git a/packages/web-crawler/src/urlRules.ts b/packages/web-crawler/src/urlRules.ts
@@ -37,7 +37,13 @@ export const crawUrlRules: CrawlUrlRule[] = [
     urlPattern: 'https://medium.com/(.*)',
     urlTransform: 'https://scribe.rip/$1',
   },
-
+  {
+    filterOptions: {
+      enableReadability: false,
+    },
+    impls: ['jina', 'browserless'],
+    urlPattern: 'https://(twitter.com|x.com)/(.*)',
+  },
   // 体育数据网站规则
   {
     filterOptions: {