sirtimbly/bewcloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Files CRUD.

Browse Source 
Remove Contacts and Calendar + CardDav and CalDav.
This commit is contained in:
Bruno Bernardino
2024-04-03 14:02:04 +01:00
parent c4788761d2
commit 4e5fdd569a
89 changed files with 2302 additions and 8001 deletions
Download Patch File Download Diff File Expand all files Collapse all files

465
lib/data/calendar.ts
View File

@@ -1,465 +0,0 @@
import { RRuleSet } from 'rrule-rust';
import Database, { sql } from '/lib/interfaces/database.ts';
import Locker from '/lib/interfaces/locker.ts';
import { Calendar, CalendarEvent, CalendarEventReminder } from '/lib/types.ts';
import { getRandomItem } from '/lib/utils/misc.ts';
import { CALENDAR_COLOR_OPTIONS, getVCalendarDate } from '/lib/utils/calendar.ts';
import { getUserById } from './user.ts';
const db = new Database();
export async function getCalendars(userId: string): Promise<Calendar[]> {
const calendars = await db.query<Calendar>(
sql`SELECT * FROM "bewcloud_calendars" WHERE "user_id" = $1 ORDER BY "created_at" ASC`,
[
userId,
],
);
return calendars;
}
export async function getCalendarEvents(
userId: string,
calendarIds: string[],
dateRange?: { start: Date; end: Date },
): Promise<CalendarEvent[]> {
if (!dateRange) {
const calendarEvents = await db.query<CalendarEvent>(
sql`SELECT * FROM "bewcloud_calendar_events" WHERE "user_id" = $1 AND "calendar_id" = ANY($2) ORDER BY "start_date" ASC`,
[
userId,
calendarIds,
],
);
return calendarEvents;
} else {
// Fetch initial recurring events and calculate any necessary to create/show for the date range, if it's not in the past
if (dateRange.end >= new Date()) {
const lock = new Locker(`events-${userId}`);
await lock.acquire();
const initialRecurringCalendarEvents = await db.query<CalendarEvent>(
sql`SELECT * FROM "bewcloud_calendar_events"
WHERE "user_id" = $1
AND "calendar_id" = ANY($2)
AND "start_date" <= $3
AND ("extra" ->> 'is_recurring')::boolean IS TRUE
AND ("extra" ->> 'recurring_id')::uuid = "id"
ORDER BY "start_date" ASC`,
[
userId,
calendarIds,
dateRange.end,
],
);
// For each initial recurring event, check instance dates, check if those exist in calendarEvents. If not, create them.
for (const initialRecurringCalendarEvent of initialRecurringCalendarEvents) {
try {
const oneMonthAgo = new Date(new Date().setUTCMonth(new Date().getUTCMonth() - 1));
let recurringInstanceStartDate = initialRecurringCalendarEvent.start_date;
let lastSequence = initialRecurringCalendarEvent.extra.recurring_sequence!;
if (recurringInstanceStartDate <= oneMonthAgo) {
// Fetch the latest recurring sample, so we don't have to calculate as many recurring dates, but still preserve the original date's properties for generating the recurring instances
const latestRecurringInstance = (await db.query<CalendarEvent>(
sql`SELECT * FROM "bewcloud_calendar_events"
WHERE "user_id" = $1
AND "calendar_id" = ANY($2)
AND "start_date" <= $3
AND ("extra" ->> 'is_recurring')::boolean IS TRUE
AND ("extra" ->> 'recurring_id')::uuid = $4
ORDER BY ("extra" ->> 'recurring_sequence')::number DESC
LIMIT 1`,
[
userId,
calendarIds,
dateRange.end,
initialRecurringCalendarEvent.extra.recurring_id!,
],
))[0];
if (latestRecurringInstance) {
recurringInstanceStartDate = latestRecurringInstance.start_date;
lastSequence = latestRecurringInstance.extra.recurring_sequence!;
}
}
const rRuleSet = RRuleSet.parse(
`DTSTART:${
getVCalendarDate(recurringInstanceStartDate)
}\n${initialRecurringCalendarEvent.extra.recurring_rrule}`,
);
const maxRecurringDatesToGenerate = 30;
const timestamps = rRuleSet.all(maxRecurringDatesToGenerate);
const validDates = timestamps.map((timestamp) => new Date(timestamp)).filter((date) => date <= dateRange.end);
// For each date, check if an instance already exists. If not, create it and add it.
for (const instanceDate of validDates) {
instanceDate.setHours(recurringInstanceStartDate.getHours()); // NOTE: Something is making the hour shift when it shouldn't
const matchingRecurringInstance = (await db.query<CalendarEvent>(
sql`SELECT * FROM "bewcloud_calendar_events"
WHERE "user_id" = $1
AND "calendar_id" = ANY($2)
AND "start_date" = $3
AND ("extra" ->> 'is_recurring')::boolean IS TRUE
AND ("extra" ->> 'recurring_id')::uuid = $4
ORDER BY "start_date" ASC
LIMIT 1`,
[
userId,
calendarIds,
instanceDate,
initialRecurringCalendarEvent.extra.recurring_id!,
],
))[0];
if (!matchingRecurringInstance) {
const oneHourLater = new Date(new Date(instanceDate).setHours(instanceDate.getHours() + 1));
const newCalendarEvent = await createCalendarEvent(
userId,
initialRecurringCalendarEvent.calendar_id,
initialRecurringCalendarEvent.title,
instanceDate,
oneHourLater,
initialRecurringCalendarEvent.is_all_day,
);
newCalendarEvent.extra = { ...newCalendarEvent.extra, ...initialRecurringCalendarEvent.extra };
newCalendarEvent.extra.recurring_sequence = ++lastSequence;
await updateCalendarEvent(newCalendarEvent);
}
}
} catch (error) {
console.error(`Error generating recurring instances: ${error}`);
console.error(error);
}
}
lock.release();
}
const calendarEvents = await db.query<CalendarEvent>(
sql`SELECT * FROM "bewcloud_calendar_events"
WHERE "user_id" = $1
AND "calendar_id" = ANY($2)
AND (
("start_date" >= $3 OR "end_date" <= $4)
OR ("start_date" < $3 AND "end_date" > $4)
)
ORDER BY "start_date" ASC`,
[
userId,
calendarIds,
dateRange.start,
dateRange.end,
],
);
return calendarEvents;
}
}
export async function getCalendarEvent(id: string, userId: string): Promise<CalendarEvent> {
const calendarEvents = await db.query<CalendarEvent>(
sql`SELECT * FROM "bewcloud_calendar_events" WHERE "id" = $1 AND "user_id" = $2 LIMIT 1`,
[
id,
userId,
],
);
return calendarEvents[0];
}
export async function getCalendar(id: string, userId: string) {
const calendars = await db.query<Calendar>(
sql`SELECT * FROM "bewcloud_calendars" WHERE "id" = $1 AND "user_id" = $2 LIMIT 1`,
[
id,
userId,
],
);
return calendars[0];
}
export async function createCalendar(userId: string, name: string, color?: string) {
const extra: Calendar['extra'] = {
default_transparency: 'opaque',
};
const revision = crypto.randomUUID();
const newColor = color || getRandomItem(CALENDAR_COLOR_OPTIONS);
const newCalendar = (await db.query<Calendar>(
sql`INSERT INTO "bewcloud_calendars" (
"user_id",
"revision",
"name",
"color",
"is_visible",
"extra"
) VALUES ($1, $2, $3, $4, $5, $6)
RETURNING *`,
[
userId,
revision,
name,
newColor,
true,
JSON.stringify(extra),
],
))[0];
return newCalendar;
}
export async function updateCalendar(calendar: Calendar) {
const revision = crypto.randomUUID();
await db.query(
sql`UPDATE "bewcloud_calendars" SET
"revision" = $3,
"name" = $4,
"color" = $5,
"is_visible" = $6,
"extra" = $7,
"updated_at" = now()
WHERE "id" = $1 AND "revision" = $2`,
[
calendar.id,
calendar.revision,
revision,
calendar.name,
calendar.color,
calendar.is_visible,
JSON.stringify(calendar.extra),
],
);
}
export async function deleteCalendar(id: string, userId: string) {
await db.query(
sql`DELETE FROM "bewcloud_calendar_events" WHERE "calendar_id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
await db.query(
sql`DELETE FROM "bewcloud_calendars" WHERE "id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
}
async function updateCalendarRevision(calendar: Calendar) {
const revision = crypto.randomUUID();
await db.query(
sql`UPDATE "bewcloud_calendars" SET
"revision" = $3,
"updated_at" = now()
WHERE "id" = $1 AND "revision" = $2`,
[
calendar.id,
calendar.revision,
revision,
],
);
}
export async function createCalendarEvent(
userId: string,
calendarId: string,
title: string,
startDate: Date,
endDate: Date,
isAllDay = false,
) {
const user = await getUserById(userId);
if (!user) {
throw new Error('User not found');
}
const calendar = await getCalendar(calendarId, userId);
if (!calendar) {
throw new Error('Calendar not found');
}
const oneHourEarlier = new Date(new Date(startDate).setHours(new Date(startDate).getHours() - 1));
const sameDayAtNine = new Date(new Date(startDate).setHours(9));
const newReminder: CalendarEventReminder = {
start_date: isAllDay ? sameDayAtNine.toISOString() : oneHourEarlier.toISOString(),
type: 'display',
};
const extra: CalendarEvent['extra'] = {
organizer_email: user.email,
transparency: 'default',
reminders: [newReminder],
};
const revision = crypto.randomUUID();
const status: CalendarEvent['status'] = 'scheduled';
const newCalendarEvent = (await db.query<CalendarEvent>(
sql`INSERT INTO "bewcloud_calendar_events" (
"user_id",
"calendar_id",
"revision",
"title",
"start_date",
"end_date",
"is_all_day",
"status",
"extra"
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
RETURNING *`,
[
userId,
calendarId,
revision,
title,
startDate,
endDate,
isAllDay,
status,
JSON.stringify(extra),
],
))[0];
await updateCalendarRevision(calendar);
return newCalendarEvent;
}
export async function updateCalendarEvent(calendarEvent: CalendarEvent, oldCalendarId?: string) {
const revision = crypto.randomUUID();
const user = await getUserById(calendarEvent.user_id);
if (!user) {
throw new Error('User not found');
}
const calendar = await getCalendar(calendarEvent.calendar_id, user.id);
if (!calendar) {
throw new Error('Calendar not found');
}
const oldCalendar = oldCalendarId ? await getCalendar(oldCalendarId, user.id) : null;
const oldCalendarEvent = await getCalendarEvent(calendarEvent.id, user.id);
if (oldCalendarEvent.start_date !== calendarEvent.start_date) {
const oneHourEarlier = new Date(
new Date(calendarEvent.start_date).setHours(new Date(calendarEvent.start_date).getHours() - 1),
);
const sameDayAtNine = new Date(new Date(calendarEvent.start_date).setHours(9));
const newReminder: CalendarEventReminder = {
start_date: calendarEvent.is_all_day ? sameDayAtNine.toISOString() : oneHourEarlier.toISOString(),
type: 'display',
};
if (!Array.isArray(calendarEvent.extra.reminders)) {
calendarEvent.extra.reminders = [newReminder];
} else {
if (calendarEvent.extra.reminders.length === 0) {
calendarEvent.extra.reminders.push(newReminder);
} else {
calendarEvent.extra.reminders[0] = { ...calendarEvent.extra.reminders[0], start_date: newReminder.start_date };
}
}
}
await db.query(
sql`UPDATE "bewcloud_calendar_events" SET
"revision" = $3,
"calendar_id" = $4,
"title" = $5,
"start_date" = $6,
"end_date" = $7,
"is_all_day" = $8,
"status" = $9,
"extra" = $10,
"updated_at" = now()
WHERE "id" = $1 AND "revision" = $2`,
[
calendarEvent.id,
calendarEvent.revision,
revision,
calendarEvent.calendar_id,
calendarEvent.title,
calendarEvent.start_date,
calendarEvent.end_date,
calendarEvent.is_all_day,
calendarEvent.status,
JSON.stringify(calendarEvent.extra),
],
);
await updateCalendarRevision(calendar);
if (oldCalendar) {
await updateCalendarRevision(oldCalendar);
}
}
export async function deleteCalendarEvent(id: string, calendarId: string, userId: string) {
const calendar = await getCalendar(calendarId, userId);
if (!calendar) {
throw new Error('Calendar not found');
}
await db.query(
sql`DELETE FROM "bewcloud_calendar_events" WHERE "id" = $1 AND "calendar_id" = $2 AND "user_id" = $3`,
[
id,
calendarId,
userId,
],
);
await updateCalendarRevision(calendar);
}
export async function searchCalendarEvents(
searchTerm: string,
userId: string,
calendarIds: string[],
): Promise<CalendarEvent[]> {
const calendarEvents = await db.query<CalendarEvent>(
sql`SELECT * FROM "bewcloud_calendar_events" WHERE "user_id" = $1 AND "calendar_id" = ANY($2) AND ("title" ILIKE $3 OR "extra"::text ILIKE $3) ORDER BY "start_date" ASC`,
[
userId,
calendarIds,
`%${searchTerm.split(' ').join('%')}%`,
],
);
return calendarEvents;
}

138
lib/data/contacts.ts
View File

@@ -1,138 +0,0 @@
import Database, { sql } from '/lib/interfaces/database.ts';
import { Contact } from '/lib/types.ts';
import { CONTACTS_PER_PAGE_COUNT } from '/lib/utils/contacts.ts';
import { updateUserContactRevision } from './user.ts';
const db = new Database();
export async function getContacts(userId: string, pageIndex: number) {
const contacts = await db.query<Pick<Contact, 'id' | 'first_name' | 'last_name'>>(
sql`SELECT "id", "first_name", "last_name" FROM "bewcloud_contacts" WHERE "user_id" = $1 ORDER BY "first_name" ASC, "last_name" ASC LIMIT ${CONTACTS_PER_PAGE_COUNT} OFFSET $2`,
[
userId,
pageIndex * CONTACTS_PER_PAGE_COUNT,
],
);
return contacts;
}
export async function getContactsCount(userId: string) {
const results = await db.query<{ count: number }>(
sql`SELECT COUNT("id") AS "count" FROM "bewcloud_contacts" WHERE "user_id" = $1`,
[
userId,
],
);
return Number(results[0]?.count || 0);
}
export async function searchContacts(searchTerm: string, userId: string, pageIndex: number) {
const contacts = await db.query<Pick<Contact, 'id' | 'first_name' | 'last_name'>>(
sql`SELECT "id", "first_name", "last_name" FROM "bewcloud_contacts" WHERE "user_id" = $1 AND ("first_name" ILIKE $3 OR "last_name" ILIKE $3 OR "extra"::text ILIKE $3) ORDER BY "first_name" ASC, "last_name" ASC LIMIT ${CONTACTS_PER_PAGE_COUNT} OFFSET $2`,
[
userId,
pageIndex * CONTACTS_PER_PAGE_COUNT,
`%${searchTerm.split(' ').join('%')}%`,
],
);
return contacts;
}
export async function searchContactsCount(search: string, userId: string) {
const results = await db.query<{ count: number }>(
sql`SELECT COUNT("id") AS "count" FROM "bewcloud_contacts" WHERE "user_id" = $1 AND ("first_name" ILIKE $2 OR "last_name" ILIKE $2 OR "extra"::text ILIKE $2)`,
[
userId,
`%${search}%`,
],
);
return Number(results[0]?.count || 0);
}
export async function getAllContacts(userId: string) {
const contacts = await db.query<Contact>(sql`SELECT * FROM "bewcloud_contacts" WHERE "user_id" = $1`, [
userId,
]);
return contacts;
}
export async function getContact(id: string, userId: string) {
const contacts = await db.query<Contact>(
sql`SELECT * FROM "bewcloud_contacts" WHERE "id" = $1 AND "user_id" = $2 LIMIT 1`,
[
id,
userId,
],
);
return contacts[0];
}
export async function createContact(userId: string, firstName: string, lastName: string) {
const extra: Contact['extra'] = {};
const revision = crypto.randomUUID();
const newContact = (await db.query<Contact>(
sql`INSERT INTO "bewcloud_contacts" (
"user_id",
"revision",
"first_name",
"last_name",
"extra"
) VALUES ($1, $2, $3, $4, $5)
RETURNING *`,
[
userId,
revision,
firstName,
lastName,
JSON.stringify(extra),
],
))[0];
await updateUserContactRevision(userId);
return newContact;
}
export async function updateContact(contact: Contact) {
const revision = crypto.randomUUID();
await db.query(
sql`UPDATE "bewcloud_contacts" SET
"revision" = $3,
"first_name" = $4,
"last_name" = $5,
"extra" = $6,
"updated_at" = now()
WHERE "id" = $1 AND "revision" = $2`,
[
contact.id,
contact.revision,
revision,
contact.first_name,
contact.last_name,
JSON.stringify(contact.extra),
],
);
await updateUserContactRevision(contact.user_id);
}
export async function deleteContact(id: string, userId: string) {
await db.query(
sql`DELETE FROM "bewcloud_contacts" WHERE "id" = $1 AND "user_id" = $2`,
[
id,
userId,
],
);
await updateUserContactRevision(userId);
}

261
lib/data/files.ts Normal file
View File

@@ -0,0 +1,261 @@
import { join } from 'std/path/join.ts';
// import Database, { sql } from '/lib/interfaces/database.ts';
import { getFilesRootPath } from '/lib/config.ts';
import { Directory, DirectoryFile, FileShare } from '/lib/types.ts';
import { sortDirectoriesByName, sortEntriesByName, sortFilesByName, TRASH_PATH } from '/lib/utils/files.ts';
// const db = new Database();
export async function getDirectories(userId: string, path: string): Promise<Directory[]> {
const rootPath = join(getFilesRootPath(), userId, path);
// const directoryShares = await db.query<FileShare>(sql`SELECT * FROM "bewcloud_file_shares"
// WHERE "parent_path" = $2
// AND "type" = 'directory'
// AND (
// "owner_user_id" = $1
// OR ANY("user_ids_with_read_access") = $1
// OR ANY("user_ids_with_write_access") = $1
// )`, [
// userId,
// path,
// ]);
const directoryShares: FileShare[] = [];
// TODO: Remove this mock test
if (path === '/') {
directoryShares.push({
id: 'test-ing-123',
owner_user_id: userId,
parent_path: '/',
name: 'Testing',
type: 'directory',
user_ids_with_read_access: [],
user_ids_with_write_access: [],
extra: {
read_share_links: [],
write_share_links: [],
},
updated_at: new Date('2024-04-01'),
created_at: new Date('2024-03-31'),
});
}
const directories: Directory[] = [];
const directoryEntries = (await getPathEntries(userId, path)).filter((entry) => entry.isDirectory);
for (const entry of directoryEntries) {
const stat = await Deno.stat(join(rootPath, entry.name));
const directory: Directory = {
owner_user_id: userId,
parent_path: path,
directory_name: entry.name,
has_write_access: true,
file_share: directoryShares.find((share) =>
share.owner_user_id === userId && share.parent_path === path && share.name === entry.name
),
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
directories.push(directory);
}
// TODO: Add directoryShares that aren't owned by this user
directories.sort(sortDirectoriesByName);
return directories;
}
export async function getFiles(userId: string, path: string): Promise<DirectoryFile[]> {
const rootPath = join(getFilesRootPath(), userId, path);
// const fileShares = await db.query<FileShare>(sql`SELECT * FROM "bewcloud_file_shares"
// WHERE "parent_path" = $2
// AND "type" = 'file'
// AND (
// "owner_user_id" = $1
// OR ANY("user_ids_with_read_access") = $1
// OR ANY("user_ids_with_write_access") = $1
// )`, [
// userId,
// path,
// ]);
const fileShares: FileShare[] = [];
const files: DirectoryFile[] = [];
const fileEntries = (await getPathEntries(userId, path)).filter((entry) => entry.isFile);
for (const entry of fileEntries) {
const stat = await Deno.stat(join(rootPath, entry.name));
const file: DirectoryFile = {
owner_user_id: userId,
parent_path: path,
file_name: entry.name,
has_write_access: true,
file_share: fileShares.find((share) =>
share.owner_user_id === userId && share.parent_path === path && share.name === entry.name
),
size_in_bytes: stat.size,
updated_at: stat.mtime || new Date(),
created_at: stat.birthtime || new Date(),
};
files.push(file);
}
// TODO: Add fileShares that aren't owned by this user
files.sort(sortFilesByName);
return files;
}
async function getPathEntries(userId: string, path: string): Promise<Deno.DirEntry[]> {
const rootPath = join(getFilesRootPath(), userId, path);
// Ensure the user directory exists
if (path === '/') {
try {
await Deno.stat(rootPath);
} catch (error) {
if (error.toString().includes('NotFound')) {
await Deno.mkdir(rootPath, { recursive: true });
}
}
}
const entries: Deno.DirEntry[] = [];
for await (const dirEntry of Deno.readDir(rootPath)) {
entries.push(dirEntry);
}
entries.sort(sortEntriesByName);
return entries;
}
export async function createDirectory(userId: string, path: string, name: string): Promise<boolean> {
const rootPath = join(getFilesRootPath(), userId, path);
try {
await Deno.mkdir(join(rootPath, name), { recursive: true });
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function renameDirectoryOrFile(
userId: string,
oldPath: string,
newPath: string,
oldName: string,
newName: string,
): Promise<boolean> {
const oldRootPath = join(getFilesRootPath(), userId, oldPath);
const newRootPath = join(getFilesRootPath(), userId, newPath);
try {
await Deno.rename(join(oldRootPath, oldName), join(newRootPath, newName));
// TODO: Update any matching file shares
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function deleteDirectoryOrFile(userId: string, path: string, name: string): Promise<boolean> {
const rootPath = join(getFilesRootPath(), userId, path);
try {
if (path.startsWith(TRASH_PATH)) {
await Deno.remove(join(rootPath, name), { recursive: true });
} else {
const trashPath = join(getFilesRootPath(), userId, TRASH_PATH);
await Deno.rename(join(rootPath, name), join(trashPath, name));
// TODO: Delete any matching file shares
}
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function createFile(
userId: string,
path: string,
name: string,
contents: string | ArrayBuffer,
): Promise<boolean> {
const rootPath = `${getFilesRootPath()}/${userId}${path}`;
try {
if (typeof contents === 'string') {
await Deno.writeTextFile(join(rootPath, name), contents, { append: false, createNew: true });
} else {
await Deno.writeFile(join(rootPath, name), new Uint8Array(contents), { append: false, createNew: true });
}
} catch (error) {
console.error(error);
return false;
}
return true;
}
export async function getFile(
userId: string,
path: string,
name: string,
): Promise<{ success: boolean; contents?: Uint8Array; contentType?: string }> {
const rootPath = `${getFilesRootPath()}/${userId}${path}`;
try {
const contents = await Deno.readFile(join(rootPath, name));
let contentType = 'application/octet-stream';
// NOTE: Detecting based on extension is not accurate, but installing a dependency like `npm:file-types` just for this seems unnecessary
const extension = name.split('.').slice(-1).join('').toLowerCase();
if (extension === 'jpg' || extension === 'jpeg') {
contentType = 'image/jpeg';
} else if (extension === 'png') {
contentType = 'image/png';
} else if (extension === 'pdf') {
contentType = 'application/pdf';
} else if (extension === 'txt' || extension === 'md') {
contentType = 'text/plain';
}
return {
success: true,
contents,
contentType,
};
} catch (error) {
console.error(error);
return {
success: false,
};
}
}

167
lib/data/news.ts
View File

@@ -1,6 +1,7 @@
import { Feed } from 'https://deno.land/x/rss@1.0.0/mod.ts';
import Database, { sql } from '/lib/interfaces/database.ts';
import Locker from '/lib/interfaces/locker.ts';
import { NewsFeed, NewsFeedArticle } from '/lib/types.ts';
import {
findFeedInUrl,
@@ -211,90 +212,98 @@ type JsonFeedArticle = JsonFeed['items'][number];
const MAX_ARTICLES_CRAWLED_PER_RUN = 10;
export async function crawlNewsFeed(newsFeed: NewsFeed) {
// TODO: Lock this per feedId, so no two processes run this at the same time
const lock = new Locker(`feeds:${newsFeed.id}`);
if (!newsFeed.extra.title || !newsFeed.extra.feed_type || !newsFeed.extra.crawl_type) {
const feedUrl = await findFeedInUrl(newsFeed.feed_url);
await lock.acquire();
if (!feedUrl) {
throw new Error(
`Invalid URL for feed: "${feedUrl}"`,
try {
if (!newsFeed.extra.title || !newsFeed.extra.feed_type || !newsFeed.extra.crawl_type) {
const feedUrl = await findFeedInUrl(newsFeed.feed_url);
if (!feedUrl) {
throw new Error(
`Invalid URL for feed: "${feedUrl}"`,
);
}
if (feedUrl !== newsFeed.feed_url) {
newsFeed.feed_url = feedUrl;
}
const feedInfo = await getFeedInfo(newsFeed.feed_url);
newsFeed.extra.title = feedInfo.title;
newsFeed.extra.feed_type = feedInfo.feed_type;
newsFeed.extra.crawl_type = feedInfo.crawl_type;
}
const feedArticles = await fetchNewsArticles(newsFeed);
const articles: Omit<NewsFeedArticle, 'id' | 'user_id' | 'feed_id' | 'extra' | 'is_read' | 'created_at'>[] = [];
for (const feedArticle of feedArticles) {
// Don't add too many articles per run
if (articles.length >= MAX_ARTICLES_CRAWLED_PER_RUN) {
continue;
}
const url = (feedArticle as JsonFeedArticle).url || getArticleUrl((feedArticle as FeedArticle).links) ||
feedArticle.id;
const articleIsoDate = (feedArticle as JsonFeedArticle).date_published ||
(feedArticle as FeedArticle).published?.toISOString() || (feedArticle as JsonFeedArticle).date_modified ||
(feedArticle as FeedArticle).updated?.toISOString();
const articleDate = articleIsoDate ? new Date(articleIsoDate) : new Date();
const summary = await parseTextFromHtml(
(feedArticle as FeedArticle).description?.value || (feedArticle as FeedArticle).content?.value ||
(feedArticle as JsonFeedArticle).content_text || (feedArticle as JsonFeedArticle).content_html ||
(feedArticle as JsonFeedArticle).summary || '',
);
}
if (feedUrl !== newsFeed.feed_url) {
newsFeed.feed_url = feedUrl;
}
const feedInfo = await getFeedInfo(newsFeed.feed_url);
newsFeed.extra.title = feedInfo.title;
newsFeed.extra.feed_type = feedInfo.feed_type;
newsFeed.extra.crawl_type = feedInfo.crawl_type;
}
const feedArticles = await fetchNewsArticles(newsFeed);
const articles: Omit<NewsFeedArticle, 'id' | 'user_id' | 'feed_id' | 'extra' | 'is_read' | 'created_at'>[] = [];
for (const feedArticle of feedArticles) {
// Don't add too many articles per run
if (articles.length >= MAX_ARTICLES_CRAWLED_PER_RUN) {
continue;
}
const url = (feedArticle as JsonFeedArticle).url || getArticleUrl((feedArticle as FeedArticle).links) ||
feedArticle.id;
const articleIsoDate = (feedArticle as JsonFeedArticle).date_published ||
(feedArticle as FeedArticle).published?.toISOString() || (feedArticle as JsonFeedArticle).date_modified ||
(feedArticle as FeedArticle).updated?.toISOString();
const articleDate = articleIsoDate ? new Date(articleIsoDate) : new Date();
const summary = await parseTextFromHtml(
(feedArticle as FeedArticle).description?.value || (feedArticle as FeedArticle).content?.value ||
(feedArticle as JsonFeedArticle).content_text || (feedArticle as JsonFeedArticle).content_html ||
(feedArticle as JsonFeedArticle).summary || '',
);
if (url) {
articles.push({
article_title: (feedArticle as FeedArticle).title?.value || (feedArticle as JsonFeedArticle).title ||
url.replace('http://', '').replace('https://', ''),
article_url: url,
article_summary: summary,
article_date: articleDate,
});
}
}
const existingArticles = await getNewsArticlesByFeedId(newsFeed.id);
const existingArticleUrls = new Set<string>(existingArticles.map((article) => article.article_url));
const previousLatestArticleUrl = existingArticles[0]?.article_url;
let seenPreviousLatestArticleUrl = false;
let addedArticlesCount = 0;
for (const article of articles) {
// Stop looking after seeing the previous latest article
if (article.article_url === previousLatestArticleUrl) {
seenPreviousLatestArticleUrl = true;
}
if (!seenPreviousLatestArticleUrl && !existingArticleUrls.has(article.article_url)) {
try {
await createsNewsArticle(newsFeed.user_id, newsFeed.id, article);
++addedArticlesCount;
} catch (error) {
console.error(error);
console.error(`Failed to add new article: "${article.article_url}"`);
if (url) {
articles.push({
article_title: (feedArticle as FeedArticle).title?.value || (feedArticle as JsonFeedArticle).title ||
url.replace('http://', '').replace('https://', ''),
article_url: url,
article_summary: summary,
article_date: articleDate,
});
}
}
const existingArticles = await getNewsArticlesByFeedId(newsFeed.id);
const existingArticleUrls = new Set<string>(existingArticles.map((article) => article.article_url));
const previousLatestArticleUrl = existingArticles[0]?.article_url;
let seenPreviousLatestArticleUrl = false;
let addedArticlesCount = 0;
for (const article of articles) {
// Stop looking after seeing the previous latest article
if (article.article_url === previousLatestArticleUrl) {
seenPreviousLatestArticleUrl = true;
}
if (!seenPreviousLatestArticleUrl && !existingArticleUrls.has(article.article_url)) {
try {
await createsNewsArticle(newsFeed.user_id, newsFeed.id, article);
++addedArticlesCount;
} catch (error) {
console.error(error);
console.error(`Failed to add new article: "${article.article_url}"`);
}
}
}
console.log('Added', addedArticlesCount, 'new articles');
newsFeed.last_crawled_at = new Date();
await updateNewsFeed(newsFeed);
} catch (error) {
lock.release();
throw error;
}
console.log('Added', addedArticlesCount, 'new articles');
newsFeed.last_crawled_at = new Date();
await updateNewsFeed(newsFeed);
}

11
lib/data/user.ts
View File

@@ -283,14 +283,3 @@ export async function validateVerificationCode(
throw new Error('Not Found');
}
}
export async function updateUserContactRevision(id: string) {
const user = await getUserById(id);
const revision = crypto.randomUUID();
user.extra.contacts_revision = revision;
user.extra.contacts_updated_at = new Date().toISOString();
await updateUser(user);
}