sirtimbly/bewcloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix news cron locks and fetching to work more than once per day.

Browse Source
This commit is contained in:
Bruno Bernardino
2024-04-19 12:19:26 +01:00
parent a8076f6420
commit 2920de90b6
4 changed files with 18 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crons/news.ts
View File

@@ -10,9 +10,9 @@ export async function fetchNewArticles(forceFetch = false) {
try { try {
const feedsToCrawl = await db.query<NewsFeed>( const feedsToCrawl = await db.query<NewsFeed>(
sql`SELECT * FROM "bewcloud_news_feeds" WHERE "last_crawled_at" IS NULL OR "last_crawled_at" <= $1`, sql`SELECT * FROM "bewcloud_news_feeds" WHERE "last_crawled_at" IS NULL OR "last_crawled_at" <= $1 ORDER BY "last_crawled_at" ASC`,
[ [
fourHoursAgo.toISOString().substring(0, 10), fourHoursAgo.toISOString(),
], ],
); );

8
lib/data/news.ts
View File

@@ -199,8 +199,8 @@ async function fetchNewsArticles(newsFeed: NewsFeed): Promise<Feed['entries'] |
return (feed as Feed)?.entries || (feed as JsonFeed)?.items || []; return (feed as Feed)?.entries || (feed as JsonFeed)?.items || [];
} catch (error) { } catch (error) {
console.log('Failed parsing feed to get articles', newsFeed.feed_url); console.error('Failed parsing feed to get articles', newsFeed.feed_url);
console.log(error); console.error(error);
} }
return []; return [];
@@ -296,11 +296,13 @@ export async function crawlNewsFeed(newsFeed: NewsFeed) {
} }
} }
console.log('Added', addedArticlesCount, 'new articles'); console.info('Added', addedArticlesCount, 'new articles');
newsFeed.last_crawled_at = new Date(); newsFeed.last_crawled_at = new Date();
await updateNewsFeed(newsFeed); await updateNewsFeed(newsFeed);
lock.release();
} catch (error) { } catch (error) {
lock.release(); lock.release();

12
lib/interfaces/locker.ts
View File

@@ -4,7 +4,7 @@ function wait(milliseconds = 100) {
return new Promise((resolve) => setTimeout(() => resolve(true), milliseconds)); return new Promise((resolve) => setTimeout(() => resolve(true), milliseconds));
} }
const expirationInMilliseconds = 15 * 1000; const expirationInMilliseconds = 15_000;
const locks: Map<string, { expiresAt: Date }> = new Map(); const locks: Map<string, { expiresAt: Date }> = new Map();
@@ -16,13 +16,20 @@ export default class Locker {
} }
public async acquire() { public async acquire() {
const currentLock = locks.get(this.lockName); console.debug('Acquiring lock:', this.lockName);
let currentLock = locks.get(this.lockName);
while (currentLock) { while (currentLock) {
// Only wait if the lock hasn't expired // Only wait if the lock hasn't expired
if (currentLock.expiresAt > new Date()) { if (currentLock.expiresAt > new Date()) {
console.debug('Waiting for lock to expire:', this.lockName);
await wait(); await wait();
} else {
// Release lock since it has expired
this.release();
} }
currentLock = locks.get(this.lockName);
} }
locks.set(this.lockName, { locks.set(this.lockName, {
@@ -31,6 +38,7 @@ export default class Locker {
} }
public release() { public release() {
console.debug('Releasing lock:', this.lockName);
locks.delete(this.lockName); locks.delete(this.lockName);
} }
} }

2
lib/utils/misc.ts
View File

@@ -121,7 +121,7 @@ export async function concurrentPromises<T>(
return results; return results;
} }
const MAX_RESPONSE_TIME_IN_MS = 10 * 1000; const MAX_RESPONSE_TIME_IN_MS = 10_000;
export async function fetchUrl(url: string) { export async function fetchUrl(url: string) {
const abortController = new AbortController(); const abortController = new AbortController();