commit 7f567a7adb9574610cd3b27fbe50647fdb7e44f2 Author: Mohammad Mahdi Date: Sat May 10 20:35:16 2025 +0330 first commit diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ebebc30 --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module CatsOfMastodonBotGo + +go 1.24.2 + +require ( + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect + golang.org/x/text v0.20.0 // indirect + gorm.io/driver/sqlite v1.5.7 // indirect + gorm.io/gorm v1.26.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..24d6053 --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I= +gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= +gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde h1:9DShaph9qhkIYw7QF91I/ynrr4cOO2PZra2PFD7Mfeg= +gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw= +gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= diff --git a/helpers/dbHelpers.go b/helpers/dbHelpers.go new file mode 100644 index 0000000..12668d6 --- /dev/null +++ b/helpers/dbHelpers.go @@ -0,0 +1 @@ +package helpers \ No newline at end of file diff --git a/helpers/postHelpers.go b/helpers/postHelpers.go new file mode 100644 index 0000000..2166f1d --- /dev/null +++ b/helpers/postHelpers.go @@ -0,0 +1,43 @@ +package helpers + +import ( + "CatsOfMastodonBotGo/models" + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "strings" +) + +func GetPosts(ctx context.Context, tag string, instance string) (error, []models.Post) { + var requestUrl = instance + "/api/v1/timelines/tag/" + tag + "?limit=40" + req, err := http.NewRequestWithContext(ctx, "GET", requestUrl, nil) + if err != nil { + log.Fatal(err) + return err, nil + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatal(err) + return err, nil + } + if resp.StatusCode != 200 || strings.Split(strings.ToLower(resp.Header.Get("Content-Type")), ";")[0] != "application/json" { + log.Fatal("Status code:", resp.StatusCode, " Content-Type:", resp.Header.Get("Content-Type")) + return err, nil + } + + var posts []models.Post = nil + err = json.NewDecoder(resp.Body).Decode(&posts) + if err != nil { + log.Fatal(err) + return err, nil + } + // defer: it basically means "do this later when the function returns" + defer resp.Body.Close() + if posts == nil { + return fmt.Errorf("no posts found for tag %s on instance %s", tag, instance), nil + } + return nil, posts +} \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..ac98e0f --- /dev/null +++ b/main.go @@ -0,0 +1,80 @@ +package main + +import ( + "CatsOfMastodonBotGo/helpers" + "CatsOfMastodonBotGo/models" + "context" + "log" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func main() { + var tag = "cat" + var instance = "https://haminoa.net" + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err, posts := helpers.GetPosts(ctx, tag, instance) + if err != nil { + log.Fatal(err) + } + log.Println("Number of fetched posts: ", len(posts)) + + db, err := gorm.Open(sqlite.Open("test.db"), &gorm.Config{}) + if err != nil { + panic("failed to connect database") + } + + // Migrate the schema + db.AutoMigrate(&models.Post{}, &models.MediaAttachment{}, &models.Account{}) + + var existingAccounts []string + var existingPosts []string + db.Model(&models.Account{}).Pluck("acc_id", &existingAccounts) + db.Model(&models.Post{}).Pluck("id", &existingPosts) + log.Println("Number of existing accounts in the database: ", len(existingAccounts)) + log.Println("Number of existing posts in the database: ", len(existingPosts)) + + var newPosts []models.Post = nil + for _, post := range posts { + + if !arrayContains(existingPosts, post.ID) { + newPosts = append(newPosts, post) + } + + } + log.Println("Number of new posts: ", len(newPosts)) + + var newAccounts []models.Account = nil + accountSet := make(map[string]bool) + for _, post := range newPosts { + if _, value := accountSet[post.Account.AccId]; !value { + accountSet[post.Account.AccId] = true + newAccounts = append(newAccounts, post.Account) + } + } + log.Println("Number of new accounts: ", len(newAccounts)) + + if newAccounts != nil { + db.Create(&newAccounts) + } else { + log.Println("No new accounts inserted") + } + if newPosts != nil { + db.Create(&newPosts) + } else { + log.Println("No new posts inserted") + } + +} + +func arrayContains(arr []string, str string) bool { + for _, a := range arr { + if a == str { + return true + } + } + return false +} diff --git a/models/account.go b/models/account.go new file mode 100644 index 0000000..3796296 --- /dev/null +++ b/models/account.go @@ -0,0 +1,11 @@ +package models + +type Account struct { + AccId string `json:"id" gorm:"primaryKey"` + Username string `json:"username"` + Acct string `json:"acct"` + DisplayName string `json:"display_name"` + IsBot bool `json:"bot"` + Url string `json:"url"` + AvatarStatic string `json:"avatar_static"` +} \ No newline at end of file diff --git a/models/mediaAttachment.go b/models/mediaAttachment.go new file mode 100644 index 0000000..d870901 --- /dev/null +++ b/models/mediaAttachment.go @@ -0,0 +1,11 @@ +package models + +type MediaAttachment struct { + ID string `json:"id" gorm:"primaryKey"` + Type string `json:"type"` + Url string `json:"url"` + PreviewUrl string `json:"preview_url"` + RemoteUrl string `json:"remote_url"` + Approved bool `json:"-"` + PostID string // Foreign key to Post +} diff --git a/models/post.go b/models/post.go new file mode 100644 index 0000000..48cebc9 --- /dev/null +++ b/models/post.go @@ -0,0 +1,9 @@ +package models + +type Post struct { + ID string `json:"id" gorm:"primaryKey"` + Url string `json:"url"` + AccountID string // Foreign key field (must match Account.AccId) + Account Account `json:"account" gorm:"foreignKey:AccountID;references:AccId"` + Attachments []MediaAttachment `json:"media_attachments" gorm:"foreignKey:PostID;references:ID"` +} diff --git a/test.db b/test.db new file mode 100644 index 0000000..947c1d1 Binary files /dev/null and b/test.db differ