mahdium/CatsOfMastodon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish migration to MongoDb and general improvements

Browse Source
This commit is contained in:
Mohammad Mahdi
2024-12-15 21:22:36 +03:30
parent 29011e34f9
commit dc38ce927f
13 changed files with 370 additions and 416 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
Services/DbInitializer.cs
View File

@@ -1,48 +1,24 @@
using JsonFlatFileDataStore;
using MongoDB.Driver;
using mstdnCats.Models;
namespace mstdnCats.Services
namespace mstdnCats.Services;
public class DbInitializer
{
public class DbInitializer
public static Task<IMongoCollection<Post>> SetupDb(string mongoDbConnectionString, string dbName)
{
public static Task<IDocumentCollection<Post>> SetupJsonDb(string _dbname)
if (mongoDbConnectionString == null) throw new Exception("MongoDb connection string is null");
try
{
// Setup DB
IDocumentCollection<Post>? collection = null;
try
{
// Initialize Backup DB
var store = new DataStore($"./data/{_dbname + "_BK"}.json", minifyJson: true);
collection = store.GetCollection<Post>();
}
catch
{
return Task.FromResult<IDocumentCollection<Post>>(null);
}
// Return collection
return Task.FromResult(collection);
var client = new MongoClient(mongoDbConnectionString);
var database = client.GetDatabase(dbName).GetCollection<Post>("posts");
return Task.FromResult(database);
}
public static Task<IMongoCollection<Post>> SetupDb(string mongoDbConnectionString, string dbName)
catch (Exception ex)
{
if (mongoDbConnectionString == null)
{
throw new Exception("MongoDb connection string is null");
}
try
{
var client = new MongoClient(mongoDbConnectionString);
var database = client.GetDatabase(dbName).GetCollection<Post>("posts");
return Task.FromResult(database);
}
catch (Exception ex)
{
throw new Exception("Error while connecting to MongoDB: " + ex.Message);
}
throw new Exception("Error while connecting to MongoDB: " + ex.Message);
}
}
}

28
Services/HandleDbBackup.cs
View File

@@ -1,5 +1,6 @@
using JsonFlatFileDataStore;
using System.Text;
using Microsoft.Extensions.Logging;
using MongoDB.Bson;
using MongoDB.Driver;
using mstdnCats.Models;
using Telegram.Bot;
@@ -10,25 +11,20 @@ namespace mstdnCats.Services;
public class HandleDbBackup
{
public static async Task HandleDbBackupAsync(TelegramBotClient _bot, ILogger<MastodonBot>? logger, string dbname, string adminId,IDocumentCollection<Post> _bkDb,IMongoCollection<Post> _db)
public static async Task HandleDbBackupAsync(TelegramBotClient _bot, ILogger<MastodonBot>? logger, string dbname,
string adminId, IMongoCollection<Post> _db)
{
logger?.LogInformation("Backup requested");
// Retrieve all posts from DB (Exclude _id field from mongoDB since it is not needed nor implemented in Post model)
var posts = _db.AsQueryable().ToList();
// Retrieve all existing posts in backup DB
var existingPosts = _bkDb.AsQueryable().ToList();
// Insert new posts that are not in backup DB (First saves all the new ones in a list and then inserts them all at once)
var newPosts = posts.Where(x => !existingPosts.Any(y => y.mstdnPostId == x.mstdnPostId)).ToList();
await _bkDb.InsertManyAsync(newPosts);
await using Stream stream = System.IO.File.OpenRead("./data/" + dbname+"_BK.json");
var message = await _bot.SendDocument(adminId, document: InputFile.FromStream(stream, dbname+"_BK.json"),
caption: "Backup of " + dbname + "\nCreated at " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss" + "\nCurrent post count: " + _db.AsQueryable().Count()), parseMode: ParseMode.Html);
var json = _db.Find(new BsonDocument()).ToList().ToJson();
var bytes = Encoding.UTF8.GetBytes(json);
var stream = new MemoryStream(bytes);
await _bot.SendDocument(adminId, InputFile.FromStream(stream, "backup.json"),
"Backup of your collection\nCreated at " +
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss" + "\nCurrent post count: " + _db.CountDocuments(new BsonDocument())),
ParseMode.Html);
logger?.LogInformation("Backup sent");
}
}

188
Services/HandlePostAction.cs
View File

@@ -1,6 +1,4 @@
using CatsOfMastodonBot.Models;
using JsonFlatFileDataStore;
using Microsoft.AspNetCore.Mvc.Diagnostics;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using mstdnCats.Models;
@@ -9,118 +7,118 @@ using Telegram.Bot.Types;
using Telegram.Bot.Types.Enums;
using Telegram.Bot.Types.ReplyMarkups;
namespace mstdnCats.Services
namespace mstdnCats.Services;
public class HandlePostAction
{
public class HandlePostAction
public static async Task HandleCallbackQuery(CallbackQuery callbackQuery, IMongoCollection<Post> _db,
TelegramBotClient _bot, ILogger<MastodonBot>? logger)
{
public static async Task HandleCallbackQuery(CallbackQuery callbackQuery, IMongoCollection<Post> _db, TelegramBotClient _bot, ILogger<MastodonBot>? logger)
var config = ConfigData.fetchData();
// Extract media ID from callback query data
string[] parts = callbackQuery.Data.Split('-');
if (parts.Length != 2)
{
var config = ConfigData.fetchData();
logger?.LogError("Invalid callback query data format.");
return;
}
// Extract media ID from callback query data
string[] parts = callbackQuery.Data.Split('-');
if (parts.Length != 2)
var action = parts[0];
var mediaId = parts[1];
var filter = Builders<Post>.Filter.Eq("MediaAttachments.MediaId", mediaId);
var post = await _db.Find(filter).FirstOrDefaultAsync();
if (post == null)
{
logger?.LogInformation("No matching post found.");
return;
}
// Approve the media attachment
if (action == "approve")
{
var mediaAttachment = post.MediaAttachments.FirstOrDefault(m => m.MediaId == mediaId);
if (mediaAttachment != null)
{
logger?.LogError("Invalid callback query data format.");
return;
}
string action = parts[0];
string mediaId = parts[1];
var filter = Builders<Post>.Filter.Eq("MediaAttachments.MediaId", mediaId);
var post = await _db.Find(filter).FirstOrDefaultAsync();
if (post == null)
{
logger?.LogInformation("No matching post found.");
return;
}
// Approve the media attachment
if (action == "approve")
{
var mediaAttachment = post.MediaAttachments.FirstOrDefault(m => m.MediaId == mediaId);
if (mediaAttachment != null)
// Check if the media attachment is already approved
if (mediaAttachment.Approved)
{
// Check if the media attachment is already approved
if (mediaAttachment.Approved)
{
await _bot.AnswerCallbackQuery(callbackQuery.Id, "Media attachment is already approved.", true);
return;
}
// Update the media attachment
var update = Builders<Post>.Update.Set("MediaAttachments.$.Approved", true);
var result = await _db.UpdateOneAsync(filter, update);
if (result.ModifiedCount > 0)
{
try
{
// Send the media attachment to the channel
var allMediaAttachments = post.MediaAttachments.ToList();
await _bot.SendPhoto(config.CHANNEL_NUMID,
allMediaAttachments.First(m => m.MediaId == mediaId).Url,
caption: $"Post from " + $"<a href=\"" + post.Account.Url + "\">" +
post.Account.DisplayName + " </a>", parseMode: ParseMode.Html
, replyMarkup: new InlineKeyboardMarkup(InlineKeyboardButton.WithUrl("View on Mastodon", post.Url)));
await _bot.AnswerCallbackQuery(callbackQuery.Id,
"Media attachment approved and sent to channel.");
await _bot.DeleteMessage(callbackQuery.Message.Chat.Id, callbackQuery.Message.Id);
logger?.LogTrace($"Media attachment {mediaId} approved.");
}
catch (Exception e)
{
logger?.LogError($"Error while sending image to the channel:{e}");
}
}
else
{
logger?.LogError($"Failed to update the media attachment {mediaId}. Record might not exist or was not found.");
}
}
else
{
logger?.LogError($"No media attachment found with MediaId {mediaId}.");
}
}
// Reject the media attachment
else if (action == "reject")
{
// Check if the post has only one attachment, if so, do not delete it, else delete the associated attachment
if (post.MediaAttachments.Count == 1 && post.MediaAttachments.First().MediaId == mediaId)
{
await _bot.AnswerCallbackQuery(callbackQuery.Id, "Post has only one attachment. No deletion performed.");
await _bot.AnswerCallbackQuery(callbackQuery.Id, "Media attachment is already approved.", true);
await _bot.DeleteMessage(callbackQuery.Message.Chat.Id, callbackQuery.Message.Id);
logger?.LogTrace($"Post {post.mstdnPostId} has only one attachment. No deletion performed.");
return;
}
else
{
var update = Builders<Post>.Update.PullFilter("MediaAttachments", Builders<MediaAttachment>.Filter.Eq("MediaId", mediaId));
var result = await _db.UpdateOneAsync(filter, update);
if (result.ModifiedCount > 0)
// Update the media attachment
var update = Builders<Post>.Update.Set("MediaAttachments.$.Approved", true);
var result = await _db.UpdateOneAsync(filter, update);
if (result.ModifiedCount > 0)
try
{
await _bot.AnswerCallbackQuery(callbackQuery.Id, "Media attachment rejected.");
// Send the media attachment to the channel
var allMediaAttachments = post.MediaAttachments.ToList();
await _bot.SendPhoto(config.CHANNEL_NUMID,
allMediaAttachments.First(m => m.MediaId == mediaId).Url,
$"Post from " + $"<a href=\"" + post.Account.Url + "\">" +
post.Account.DisplayName + " </a>", ParseMode.Html
, replyMarkup: new InlineKeyboardMarkup(InlineKeyboardButton.WithUrl("View on Mastodon", post.Url)));
await _bot.AnswerCallbackQuery(callbackQuery.Id,
"Media attachment approved and sent to channel.");
await _bot.DeleteMessage(callbackQuery.Message.Chat.Id, callbackQuery.Message.Id);
logger?.LogTrace($"Media attachment {mediaId} removed from post {post.mstdnPostId}.");
logger?.LogTrace($"Media attachment {mediaId} approved.");
}
else
catch (Exception e)
{
logger?.LogError($"Failed to remove media attachment {mediaId} from post {post.mstdnPostId}.");
logger?.LogError($"Error while sending image to the channel:{e}");
}
}
else
logger?.LogError(
$"Failed to update the media attachment {mediaId}. Record might not exist or was not found.");
}
else
{
logger?.LogError("Invalid action specified.");
logger?.LogError($"No media attachment found with MediaId {mediaId}.");
}
}
// Reject the media attachment
else if (action == "reject")
{
// Check if the post has only one attachment, if so, do not delete it, else delete the associated attachment
if (post.MediaAttachments.Count == 1 && post.MediaAttachments.First().MediaId == mediaId)
{
await _bot.AnswerCallbackQuery(callbackQuery.Id,
"Post has only one attachment. No deletion performed.");
await _bot.DeleteMessage(callbackQuery.Message.Chat.Id, callbackQuery.Message.Id);
logger?.LogTrace($"Post {post.mstdnPostId} has only one attachment. No deletion performed.");
}
else
{
var update = Builders<Post>.Update.PullFilter("MediaAttachments",
Builders<MediaAttachment>.Filter.Eq("MediaId", mediaId));
var result = await _db.UpdateOneAsync(filter, update);
if (result.ModifiedCount > 0)
{
await _bot.AnswerCallbackQuery(callbackQuery.Id, "Media attachment rejected.");
await _bot.DeleteMessage(callbackQuery.Message.Chat.Id, callbackQuery.Message.Id);
logger?.LogTrace($"Media attachment {mediaId} removed from post {post.mstdnPostId}.");
}
else
{
logger?.LogError($"Failed to remove media attachment {mediaId} from post {post.mstdnPostId}.");
}
}
}
else
{
logger?.LogError("Invalid action specified.");
}
}
}

51
Services/HandleStartMessage.cs
View File

@@ -1,42 +1,41 @@
using JsonFlatFileDataStore;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using MongoDB.Driver.Linq;
using mstdnCats.Models;
using Telegram.Bot;
using Telegram.Bot.Types;
using Telegram.Bot.Types.Enums;
using Telegram.Bot.Types.ReplyMarkups;
namespace CatsOfMastodonBot.Services
namespace CatsOfMastodonBot.Services;
public class HandleStartMessage
{
public class HandleStartMessage
public static async Task HandleStartMessageAsync(Message message, TelegramBotClient _bot,
IMongoCollection<Post> _db, ILogger<MastodonBot>? logger, CallbackQuery callbackQuery = null)
{
public static async Task HandleStartMessageAsync(Message message, TelegramBotClient _bot, IMongoCollection<Post> _db, ILogger<MastodonBot>? logger, CallbackQuery callbackQuery = null)
{
logger?.LogInformation("Start message received, trigger source: " + (callbackQuery != null ? "Callback" : "Start command"));
logger?.LogInformation("Start message received, trigger source: " +
(callbackQuery != null ? "Callback" : "Start command"));
// choose all media attachments that are approved
var mediaAttachmentsToSelect =await _db.AsQueryable()
.Where(post => post.MediaAttachments.Any(media => media.Approved))
.ToListAsync();
// select random approved media attachment
var selectedMediaAttachment = mediaAttachmentsToSelect[new Random().Next(mediaAttachmentsToSelect.Count)];
// send media attachment
await _bot.SendPhoto(message.Chat.Id, selectedMediaAttachment.MediaAttachments.FirstOrDefault(m => m.Approved == true).Url,
caption: $"Here is your cat!🐈\n" + "<a href=\"" + selectedMediaAttachment.Url + "\">" + $"View on Mastodon " + " </a>", parseMode: ParseMode.Html
, replyMarkup: new InlineKeyboardMarkup().AddButton(InlineKeyboardButton.WithUrl("Join channel 😺", "https://t.me/catsofmastodon"))
.AddNewRow()
.AddButton(InlineKeyboardButton.WithCallbackData("Send me another one!", $"new_random")));
// choose all media attachments that are approved
var mediaAttachmentsToSelect = _db.AsQueryable()
.Where(post => post.MediaAttachments.Any(media => media.Approved))
.ToList();
// answer callback query from "send me another cat" button
if (callbackQuery != null)
{
await _bot.AnswerCallbackQuery(callbackQuery.Id, "Catch your cat! 😺");
}
logger?.LogInformation("Random cat sent!");
// select random approved media attachment
var selectedMediaAttachment = mediaAttachmentsToSelect[new Random().Next(mediaAttachmentsToSelect.Count)];
// send media attachment
await _bot.SendPhoto(message.Chat.Id,
selectedMediaAttachment.MediaAttachments.FirstOrDefault(m => m.Approved == true).Url,
$"Here is your cat!🐈\n" + "<a href=\"" + selectedMediaAttachment.Url + "\">" +
$"View on Mastodon " + " </a>", ParseMode.Html
, replyMarkup: new InlineKeyboardMarkup()
.AddButton(InlineKeyboardButton.WithUrl("Join channel 😺", "https://t.me/catsofmastodon"))
.AddNewRow()
.AddButton(InlineKeyboardButton.WithCallbackData("Send me another one!", $"new_random")));
// answer callback query from "send me another cat" button
if (callbackQuery != null) await _bot.AnswerCallbackQuery(callbackQuery.Id, "Catch your cat! 😺");
}
logger?.LogInformation("Random cat sent!");
}
}

59
Services/PostResolver.cs
View File

@@ -2,40 +2,39 @@ using System.Text.Json;
using Microsoft.Extensions.Logging;
using mstdnCats.Models;
namespace mstdnCats.Services
namespace mstdnCats.Services;
public sealed class PostResolver
{
public sealed class PostResolver
public static async Task<List<Post>?> GetPostsAsync(string tag, ILogger<MastodonBot>? logger, string instance)
{
public static async Task<List<Post>?> GetPostsAsync(string tag, ILogger<MastodonBot>? logger, string instance)
// Get posts
var _httpClient = new HttpClient();
// Get posts from mastodon api (40 latest posts)
var requestUrl = $"{instance}/api/v1/timelines/tag/{tag}?limit=40";
var response = await _httpClient.GetAsync(requestUrl);
// Print out ratelimit info
logger?.LogInformation("Remaining requests: " +
response.Headers.GetValues("X-RateLimit-Remaining").First() + "time to reset: " +
response.Headers.GetValues("X-RateLimit-Reset").First());
// Check if response is ok
if (
response.StatusCode == System.Net.HttpStatusCode.OK ||
response.Content.Headers.ContentType.MediaType.Contains("application/json") ||
(response.Headers.TryGetValues("X-RateLimit-Remaining", out var remaining) &&
int.Parse(remaining.First()) != 0)
)
{
// Get posts
HttpClient _httpClient = new HttpClient();
// Get posts from mastodon api (40 latest posts)
string requestUrl = $"{instance}/api/v1/timelines/tag/{tag}?limit=40";
var response = await _httpClient.GetAsync(requestUrl);
// Deserialize response based on 'Post' model
return JsonSerializer.Deserialize<List<Post>>(await response.Content.ReadAsStringAsync());
}
// Print out ratelimit info
logger?.LogInformation("Remaining requests: " +
response.Headers.GetValues("X-RateLimit-Remaining").First() + "time to reset: " +
response.Headers.GetValues("X-RateLimit-Reset").First());
// Check if response is ok
if (
response.StatusCode == System.Net.HttpStatusCode.OK ||
response.Content.Headers.ContentType.MediaType.Contains("application/json") ||
response.Headers.TryGetValues("X-RateLimit-Remaining", out var remaining) &&
int.Parse(remaining.First()) != 0
)
{
// Deserialize response based on 'Post' model
return JsonSerializer.Deserialize<List<Post>>(await response.Content.ReadAsStringAsync());
}
else
{
logger?.LogCritical("Error while getting posts: " + response.StatusCode);
return null;
}
else
{
logger?.LogCritical("Error while getting posts: " + response.StatusCode);
return null;
}
}
}

79
Services/ProcessPosts.cs
View File

@@ -1,5 +1,4 @@
using CatsOfMastodonBot.Models;
using JsonFlatFileDataStore;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using mstdnCats.Models;
@@ -7,55 +6,53 @@ using Telegram.Bot;
using Telegram.Bot.Types.Enums;
using Telegram.Bot.Types.ReplyMarkups;
namespace mstdnCats.Services
namespace mstdnCats.Services;
public class ProcessPosts
{
public class ProcessPosts
public static async Task<List<MediaAttachment>> checkAndInsertPostsAsync(IMongoCollection<Post> _db,
TelegramBotClient _bot, List<Post> fetchedPosts, ILogger<MastodonBot>? logger)
{
public static async Task<List<MediaAttachment>> checkAndInsertPostsAsync(IMongoCollection<Post> _db, TelegramBotClient _bot, List<Post> fetchedPosts, ILogger<MastodonBot>? logger)
{
var config = ConfigData.fetchData();
var config = ConfigData.fetchData();
// Get existing posts
var existingPosts = _db.AsQueryable().Select(x => x.mstdnPostId).ToArray();
logger?.LogInformation($"Recieved posts to proccess: {fetchedPosts.Count} - total existing posts: {existingPosts.Length}");
int newPosts = 0;
// Process posts
foreach (Post post in fetchedPosts)
// Get existing posts
var existingPosts = _db.AsQueryable().Select(x => x.mstdnPostId).ToArray();
logger?.LogInformation(
$"Recieved posts to proccess: {fetchedPosts.Count} - total existing posts: {existingPosts.Length}");
var newPosts = 0;
// Process posts
List<Post> validPosts = new();
foreach (var post in fetchedPosts)
// Check if post already exists
if (!existingPosts.Contains(post.mstdnPostId) && post.MediaAttachments.Count > 0 &&
post.Account.IsBot == false)
{
// Check if post already exists
if (!existingPosts.Contains(post.mstdnPostId) && post.MediaAttachments.Count > 0 && post.Account.IsBot == false)
{
// Send approve or reject message to admin
foreach (var media in post.MediaAttachments)
{
if (media.Type == "image")
// Send approve or reject message to admin
foreach (var media in post.MediaAttachments)
if (media.Type == "image")
try
{
try
{
await _bot.SendPhoto(config.ADMIN_NUMID, media.PreviewUrl, caption: $"<a href=\"" + post.Url + "\"> Mastodon </a>", parseMode: ParseMode.Html
, replyMarkup: new InlineKeyboardMarkup().AddButton("Approve", $"approve-{media.MediaId}").AddButton("Reject", $"reject-{media.MediaId}"));
}
catch (System.Exception ex)
{
logger?.LogError("Error while sending message to admin: " + ex.Message + " - Media URL: " + media.PreviewUrl);
}
await _bot.SendPhoto(config.ADMIN_NUMID, media.PreviewUrl,
$"<a href=\"" + post.Url + "\"> Mastodon </a>", ParseMode.Html
, replyMarkup: new InlineKeyboardMarkup().AddButton("Approve", $"approve-{media.MediaId}").AddButton("Reject", $"reject-{media.MediaId}"));
validPosts.Add(post);
}
catch (Exception ex)
{
logger?.LogError("Error while sending message to admin: " + ex.Message + " - Media URL: " +
media.PreviewUrl);
}
}
// Insert post
await _db.InsertOneAsync(post);
newPosts++;
}
newPosts++;
}
logger?.LogInformation($"Proccesing done, stats: received {fetchedPosts.Count} posts, inserted and sent {newPosts} new posts.");
// Insert post
await _db.InsertManyAsync(validPosts);
logger?.LogInformation(
$"Proccesing done, stats: received {fetchedPosts.Count} posts, inserted and sent {newPosts} new posts.");
// Return list of media attachments
var alldbpostsattachmentlist = _db.AsQueryable().SelectMany(x => x.MediaAttachments).ToList();
return alldbpostsattachmentlist;
}
// Return list of media attachments
var alldbpostsattachmentlist = _db.AsQueryable().SelectMany(x => x.MediaAttachments).ToList();
return alldbpostsattachmentlist;
}
}

39
Services/RunCheck.cs
View File

@@ -1,34 +1,31 @@
using JsonFlatFileDataStore;
using Microsoft.Extensions.Logging;
using MongoDB.Driver;
using mstdnCats.Models;
using Telegram.Bot;
namespace mstdnCats.Services
namespace mstdnCats.Services;
public class RunCheck
{
public class RunCheck
public static async Task<bool> runAsync(IMongoCollection<Post> _db, TelegramBotClient _bot, string _tag,
ILogger<MastodonBot>? logger, string _instance)
{
public static async Task<bool> runAsync(IMongoCollection<Post> _db, TelegramBotClient _bot, string _tag, ILogger<MastodonBot>? logger, string _instance)
// Run check
try
{
// Run check
try
{
// First get posts
var posts = await PostResolver.GetPostsAsync(_tag, logger, _instance);
// First get posts
var posts = await PostResolver.GetPostsAsync(_tag, logger, _instance);
if (posts == null)
{
logger?.LogCritical("Unable to get posts");
}
if (posts == null) logger?.LogCritical("Unable to get posts");
// Then process them
await ProcessPosts.checkAndInsertPostsAsync(_db, _bot, posts, logger);
}
catch (Exception ex)
{
logger?.LogCritical("Error while running check: " + ex.Message);
}
return true;
// Then process them
await ProcessPosts.checkAndInsertPostsAsync(_db, _bot, posts, logger);
}
catch (Exception ex)
{
logger?.LogCritical("Error while running check: " + ex.Message);
}
return true;
}
}