From 0533151c1b517e8098652dab5ddb2cc6c3c7f38c Mon Sep 17 00:00:00 2001 From: Mohammad Mahdi Date: Sat, 10 May 2025 21:33:50 +0330 Subject: [PATCH] Refactor database operations and move helpers to separate package --- helpers/dbHelpers.go | 60 ++++++++++++++++++++++++++++++++++++++++++- main.go | 38 ++++++++------------------- test.db | Bin 57344 -> 0 bytes 3 files changed, 69 insertions(+), 29 deletions(-) delete mode 100644 test.db diff --git a/helpers/dbHelpers.go b/helpers/dbHelpers.go index 12668d6..5809ffa 100644 --- a/helpers/dbHelpers.go +++ b/helpers/dbHelpers.go @@ -1 +1,59 @@ -package helpers \ No newline at end of file +package helpers + +import ( + "CatsOfMastodonBotGo/models" + "gorm.io/gorm" +) + +func GetExistingPostIds(db *gorm.DB) []string { + var existingPostIds []string + db.Model(&models.Post{}).Pluck("id", &existingPostIds) + return existingPostIds +} + +func GetExistingAccountIds(db *gorm.DB) []string { + var existingAccountIds []string + db.Model(&models.Account{}).Pluck("acc_id", &existingAccountIds) + return existingAccountIds +} + +func GetNewPosts(existingPostIds []string, posts []models.Post) []models.Post { + var newPosts []models.Post = nil + for _, post := range posts { + if !arrayContains(existingPostIds, post.ID) { + newPosts = append(newPosts, post) + } + } + return newPosts +} + +func GetNewAccounts(existingAccountIds []string, posts []models.Post) []models.Account { + var newAccounts []models.Account = nil + for _, post := range posts { + if !arrayContains(existingAccountIds, post.Account.AccId) { + newAccounts = append(newAccounts, post.Account) + } + } + return newAccounts +} + +func InsertNewPosts(db *gorm.DB, newPosts []models.Post) int{ + return int(db.Create(&newPosts).RowsAffected) +} + +func InsertNewAccounts(db *gorm.DB, newAccounts []models.Account) int{ + return int(db.Create(&newAccounts).RowsAffected) +} + +func arrayContains(arr []string, str string) bool { + for _, a := range arr { + if a == str { + return true + } + } + return false +} + + + + diff --git a/main.go b/main.go index ac98e0f..1f60865 100644 --- a/main.go +++ b/main.go @@ -6,19 +6,18 @@ import ( "context" "log" "time" - "gorm.io/driver/sqlite" "gorm.io/gorm" ) func main() { - var tag = "cat" + var tag = "catsofmastodon" var instance = "https://haminoa.net" ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() err, posts := helpers.GetPosts(ctx, tag, instance) if err != nil { - log.Fatal(err) + panic(err) } log.Println("Number of fetched posts: ", len(posts)) @@ -30,40 +29,23 @@ func main() { // Migrate the schema db.AutoMigrate(&models.Post{}, &models.MediaAttachment{}, &models.Account{}) - var existingAccounts []string - var existingPosts []string - db.Model(&models.Account{}).Pluck("acc_id", &existingAccounts) - db.Model(&models.Post{}).Pluck("id", &existingPosts) - log.Println("Number of existing accounts in the database: ", len(existingAccounts)) - log.Println("Number of existing posts in the database: ", len(existingPosts)) + var existingPostIds = helpers.GetExistingPostIds(db) + var existingAccountIds = helpers.GetExistingAccountIds(db) + var newAccounts = helpers.GetNewAccounts(existingAccountIds, posts) + var newPosts = helpers.GetNewPosts(existingPostIds, posts) - var newPosts []models.Post = nil - for _, post := range posts { - - if !arrayContains(existingPosts, post.ID) { - newPosts = append(newPosts, post) - } - - } + log.Println("Number of existing posts: ", len(existingPostIds)) + log.Println("Number of existing accounts: ", len(existingAccountIds)) log.Println("Number of new posts: ", len(newPosts)) - - var newAccounts []models.Account = nil - accountSet := make(map[string]bool) - for _, post := range newPosts { - if _, value := accountSet[post.Account.AccId]; !value { - accountSet[post.Account.AccId] = true - newAccounts = append(newAccounts, post.Account) - } - } log.Println("Number of new accounts: ", len(newAccounts)) if newAccounts != nil { - db.Create(&newAccounts) + log.Println("Number of inserted accounts: ", db.Create(&newAccounts).RowsAffected) } else { log.Println("No new accounts inserted") } if newPosts != nil { - db.Create(&newPosts) + log.Println("Number of inserted posts: ", db.Create(&newPosts).RowsAffected) } else { log.Println("No new posts inserted") } diff --git a/test.db b/test.db deleted file mode 100644 index 947c1d1cc8910f06b4c92c2cbc67f182eddfe5d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 57344 zcmeI5dyHJyec!p{lH4WtAz7AbnWi-IOCoITeZOcMUP`4NmZ%3MT2^eCywBx`J2T6f z*(H~*>+C3noxrgY)OOmsE!)Je)QJ&9Hju^!f~GCn|N2J(w*^w5a9|fO(uPIaqJUAL zpL5^$E;(Fs$3W{`O0zqAALsEqzw#c^JFlf8y+~Lsm zFLc80aIo#dfyd6AI{DJ6)=MWJedbi_{I>h&TMwUa2j^SE@XGK5=ZBkXVfKB!znXnr z>xUcd@bbCRrGD7y4a4#kcWtfT+XyMEyWR==ZT|!3*Ls8DxxDn}&pdVd-)X+?w#~Md9mL2QhC+xr#A!z_$ugq{P^+d14ll5 zD;(A2>r$Bs>no{oE)m6rY9TkUgpS#&$CE#;l{LD=uQot%HTJIoj; zXb;v_-OY35`|ZIwug4QPx0GqO;SSyYxxvsKw*9n_D*L2$A3rYbJ#gf)V>jz**4%SS z_D!7SXT+Qmhy1C^-aEu{-~__0&DH$^*^=Gf!`QLXjQAQ4LGe4aasgMK_;dww!!nz1_{fEO!ROpu05a z`E7SKzj|RfTpOH_r1DK^dHZc46qYWyop!h9E_K79oTnmJHThKfHeJZ~po6uUSNzASv76b)@9W; zWnI;^Ztt=?yb${Cu=rN=$6xKO2hZ|Ii)UIqbp6`j{XyRDtYh(0(sEHi{>09ECQCF- zqGb|IlW3bnLBztjIxmUY4W%_P%Q)6`8vd9AnETOaVV8xF*;;*WZ7Kib>O z8(bf_%8U6^JMNaEN;FHNT@nqGXqiOQif(y+WGRj!BX}LzGj!3d(Sey+#dz_B*-&&# zHVoNTOiPs&o2T?MZa3(49(VoW;H1}EA2L*f8(mu-_Ig7x;X3PGR@>x}R{Y+HmUxNr z{Eg4YKYsMfzy71&`)by?>U+}ijSA16-T8zR`vyan=!ryEB>IB9*9$@**jb@$SvIst zk1Q|YbdSKW>F(6vinml>tD*Y3y5!=G>c!#^KwWsNQ0k(NvEWDo4TH|ZurtOT@2qCpZZ zDtaRas%)y7uGp@rXuenx*quK!%}L|rkyw(lBq+M0TaKk_rf$f(zuJzX%WPHiFU#$2 z)Ek8UQh3F`;C5HS$HXI9W9v6rw6hYjM{c7(Fd3T-+9S~*i55vTspyXuL`oogI=cx~ z(Y!Doos0X}G#P`LCsIZ;HCs1KMK)zy)lU1*wpYUBXGui*yi3Urwxah56 z3c(P#nq!8#;fsCy%9gyBqt)>vGapQPi-=WC78w4ShJCWr?)2gx^cC&PQq+(C%G;J+ zl$NWPZ>z~BgQJ@9sz$Ad{?d-nvy?FObyIURUpKT^^lw>HY{;dfxg!wihy*0gD!*%aTFV`T_wRQaouEIzQ>E#+SGoM|qq8@We*>&LHaQbvj4Sy(kx~{dJO)l59 zzB9??x(@VUT6Rrkcp!PYu72B}T&}CJrt?B7>Y#nu<*MFf@4fq!S;5F~RY^B7&op~@ z>e!ye-(5Vr@Uw-lE_`_YC-a}3zjy8r=gu7dpNGG4_~fAvTrUflNw`#!$+|LncC_vF-np86~W{v!N4|M0Z( z86}3nO%R38*NU6)VPri3rSxtK8oK^-&`I?Q{kVgMlQ1f{c zxt%uC@?sm6xU~GF+rG%O4=?tT!o@4FG+PN^Zdj(a`ftr0-LGuDm=tfoQJSjqq-E(D zO}xRy%_Xnj4whCnrM^_Tj18lN6=T$9XoGfeVr|twv1(je83tDZ-Bd4`{#E0GH&l%< zxF`wpIZ0dXUF^G;dRJC`HV|Tgsq8wo=H5|ijtV>`n-)eZ%FnSmgHGS@Zrq%g@0(UG zCbThajV+C=uxGLIp}l{@@lJsc*jPu>x{rU}E3`nK-u+oqM3ge(jYR(Tk;IC3#Vw_}q@ zvEpGJI|jjEGT&>Pnpf}5C6*12gV@0&KvB0h`f{(o8VF;m_8@Fy@|V&ic?s#FvPY)) z+OG4L-a4&ZN?I)zPDN!Uv5?6*O)3&AHxo>np1gs)un<|I=~a`&m+#3L#nHvOjA%mu z<#Y5#W_oGCjE$K0V$+~gaBcJZ@>@y;(_)SZtjSs~4969$%E=N8^t8R;Pu`t#h+|W= z2Gg=)=B%A_NM<)xS&hy39Ye#ONwvggqGta0>RtIzVl^ZrO1!FW=kJ>=KD?x0O(}Q8(FQzLhzT$#`>$U^GXX9k0Ke6 zmTc>{Ek3W#WrSvuBfb^Z1_%Dn3cb0>RZ@KXk|>ppXR_<8*m-^8aJKd!62vSw00k{I z=j&};lA1xKg^O1>4Ld4337S?jxcbzgObA+r*sdWok%VS>=TkNrm6mG@XYv+XHH0ma zdE3J2&Q2?zjVC-$B99f8LN+ruZ+yiRDCt7{B4<|)=VId54|5l0SgsByb+GD#TC>_3 zs%mER--)eFQd9goM+T0_9ND*71%(HxX@6W3at>+BiC$s5oln@hq03s`)CD{p=_L%0 z*{5TH%xH=QxN38A_CO}uQBzo!sxk{qy^z5-5$)n()3$9`)WnWyn`r;!{!H4iM>GwX zpS>AEFC)|V-Ffnkm@&!8qP{!-)O5y3uo-Ixyk;@8@_`;>mJ!oFeoIFRs*nSs)X&c! z-y**DY7Q?92-0GCSg}pWrz3Yo zQ!D3xf&V`|*WNSNp8Msw-DoEPH#pN>m^-zAf0$RPVxGZ3!3~G9Bak-2Uy(ha| z1$W+^UlveH4$Qo(xLn3n9xN`G@sJGvM~7VoGgAB?rCSAvNb!GhxeO&F_`lFAR}g>% z{}-1ln)?+07njQ#^c4T+az&S&;s2ivNqtWj#oO|BFH@>Wc*bk1v;X4+;JsUoNW;g#Z8a;$MhKy|DPR#s9VV-xvR4 z|396!_I_jPkvp8}zqqfBHVz$Dj%=-Lxm*5Bsvfiy)x>#GpnFccF?1A|XVq!+`oe5V_|ZKH z;cYbMvLHK}Wo-h4$2XS(arVSl7dJD?EVU6upXu3hq}Y*ZMIr#i?g)Q_$`Zo6V`{2v zf(g9Phux%(bZ!HV5ts%#T7VPOeFDsu5Vjx|hM_+Kz8yNDq@MO4JAG;G(Vl$es?=UT zHBc@a&ul(nZoG0qH=gNsl|;()bVmcd00UHf7n3|)aSM`{b@ijcK|%8NmcR8%N^&qMEDUWN z1p(@IFMMbl1>(1E68Ku&p_tJtYwznb2QtZXFrzwHi;1Za8(DKQ zRHr~Vg0Lp0Dp6*^8G(bvl_v*E7!d}mlnpn;iqe#K zFY1Y}Q>!c$J7a1}H$9od@hwC>HQ?(E-WSJ^aOhIL-SIrbv)DplsNf@Y{DS1=z4~Zl zUnc5vPG{3@vWpTBvNaJ-r$ESUNfe1pC`m>P2K-1PN|H;603#4>40pNTVg0Vvme70$ zRaJ1!Td}$bLxTAqti%$=b}py8;S;+n0_Trh9f>c(kc7#$rUIspHum0#@Ek_5>aeuQ zNP9sFN+iyZE>ULDNDI-7$qUB*iWd>2VPb?Y*Om}|2UXoR;Z3${V$VL%*4Tb{$YO1-OrE1Z{^3KLjwOl zK6hr%+?m({0PG(NfbY!xR*e520{?#w{Qom!EC9g&7k-#p03`VT`Nf}N0q_F~ZvHhC zXeiK7prJrRfrbJN1sV!86lf^$3r>LyFvmI>6qi`WI4D}$@!~Q#I<6GBTsgK`;Qu1d zG%w&VgHdyi|AV9BBZwaWHu#R3q#z92O@qcj{18S6X_yiaQ{*OgH&J%dA zbI&Z{|H281N-?H(s<}Lskm>A7@q_iU6u{el75S;r_%Z zqJW~X68;b5Bn%87yJM8_fBXex6LWn=UM}JPAo{VJEJfQn{*T`Y&VX3w6JbEJbNnAf z-^K?Pk6hqwv%vo~oK;Lh4&Vl6sRjP8Qy*T$=z`HM7x+IOrV7;J!)O%vzwrNnr*J%g zjs^Y?#$n;+q2gv>s|EfKu2o>XIsRWoAandbM)Y$Wtib;(SWSWd>t!&Z!2j{hC=}ZT z{$EkNmhgWFnkxrO_&=9()m0V$&lM}e|NkBW`!7De@OS3F-o*cJ;{P}C|3yB7CjS4e z6K6E>|I+}qP5l3Il7J@uzcA)%;{V6VbDH@7P5l3=FLEtZf9=mE{(qb!po#yVW#?+* z|C5xfiT~fk|G)KqJWjT-^Z5U=Fd!G_|M}_vuxGKg@aTMd?jIfgokQQ5{n6|LGiML} z%>%!?|KD({`PWdOp+G}{|F0;p)oINt$4@;m^Kx=VuZHifM(V8owOonnS-nRpoj2b6 z%RhR9%v&O7XJ#T+d7~Z&GihY zwIoH*ryd}I(_`A~S^B#z3TP>6Xz6PySx_AJD!$bUNcGerCvBW!EN^^0LRP7m{F$3L zJZ;aoiaowaP7`NbWq$|YgTqZJQB7ck4xxDP(ldQuRf0GN()0I|Z0oUOGjRv&QAo3p zrP=T0?QuWQ>d#cAW%tlW^86%<=FYvVDx(K{|-M z6eW4^K6Z?>PS&ZJr^J}mBZwx!Mw6e*)kvhLi9^~t&G~vbSijhAoe*&$&kZ`bk!|Lg z#Hw$I^Mr_9dIud73S#n+>#aLweifaWJkvn>luCnaASE zR|{cU3VB-kDwq+kveU#v71`0w#351h=BCLqOV8)GcFc8bk7x7AZPLZjxbtVAJ9U&wFXP8R^} zF!UrXA*S2-$&rz-EE#CLtht62z)ygi7=4n__Z_6uvK~pftQI>pjdUje)sSja#l9>6PaKcqZ#QkvO180TcJ z1y_BobFtUIKy29VQq->ev0M)F`cb#r4O^#|T2H4>XN@mEG%gZsrRePC+Zb+<-&ka- z#Sw)Ja0EF;o|JlSgYe`1qj-Vom~QOqL>|8WL)z*xN{Ydh*>#O2T?%*Geqd${)odP0-jU-LBhx5bpiL_wzPqEA#f4 zviT!BmM^$W5ONk53WF5eh)8~u2B28Qz@Jq`C0RIbL!KQbSuj6wuSgx~kTIN9tR7Uh z6oj_)wbH3aMV#8!^Kqcs)`=4&#vHZ>!@Tj8{*}teAJ}>ClI2S{<`CRA_JtywLD4%; z*GbB&f&I&lOJ0O{^1b?2kwg50X~N`M@Yy0(ZSk$>TO81~xWTZ!@!jj!zVfx}*M9w1 z^VVnG+ZO#ce`3e|V}YZ6`2E6^FeV`<Y#{d5g_dAXMe-_}2s5e+;H~#+!8R(Elu^(+T z{{Puw;^*f7f1`*O$@%|7dlr6We&+BShh}H~9_RnBO&{O;`%@pf$tk$GoKj%aeQOpO z=fxx-5soP(hnU?5tsqJW1p-x|Fhw_qlTD$mQryT0Ep+5o=;BTl2Tj`5hWrMVB^&Z_ zHTgkTUFx*m^`*6SZ~Tc)l#fD@oIZ34O)$}*@{KOuUj~6HT+?)oAarBZn}`w;6Eshc zG}^o;ixLuM1TVW9O@!bSNYpSz!h4BNDWXPb`9UB%KAk}SX-yzJ3lyo>mPQE~S5+J? za;b;`mE z6hI&35zflF9t@wrXHIC!YN(=lE|b2LUTd^@cSib@?W;a>yC*$5f1L1;F+wxWr)+JY z0)@)9Yr*kB=sQB^zsqn?H>fQk{n)B%*}rxYbdib(7z{EakTx@p%9Ybz_X#XyM80Vg z%}};P(8Yx9s}Y7r8+T8urr#cqNNjOE4~RF=y4dQGL^vUebGL_LJK#Qjww?NU|NJE056LO&ynJbXFs zl#un+7|x^4Vn+BE5?N1vwUX=S1ehL^4G{}bKqX+ZxRVw$)3betL~%spQiw!r3!%8X z5NHsjURgr;Ti28rMvPxuMeziXB+pY!Zxi#OoYOr;i6WTX3-C!#=*ST<7FFL4WAm8Z&8qE{W%RyzO=(&^ zzt>q?ABKJKbOJNS@kYr9+Qk&Ce~VENfSZu<1VUp(NLN|(P>uY^_ku{IRt(^-d=x6N z$VVIV*(hYDS!$$umWyQr{IT2p z4TqarZ3*qiR&|H8ho#V^!1@GOFSET5(9iD1i0qIBOe^hg1YhS_Gn4cGfjtXv%-?tT z*A5-w{C{-+H~0Owy;r9W{Pj8Lk1pMxiQP1|b~S* zBO$g|OBcvrzZBa*cEce?SQn)${WV`=54hx#J-s*VEe$ri{?bY~d~~(veI#fHALesn zZPh=qYFt_w23G>Y99=T~tHuRys2X8#@qIoId2V#EAGY1jrSQ^awXF}`E3d7(>s@bX z>239DxUtgdd2U~|Uh7_Nlc~NlxY}R2yb`uM>+xi%B>x#*I+`t!XH(5rB{<2XVSlQw%2+L!IMB||s1s@_I#DYoAfN8j;Valsp?cZffQ z){9`=Wy@mMP<{5lgvQka(iw=j=c&f4l0s?p>33uTs*}>Z5_xF?G#7UD$ySiCMKrdw zE%BhaPmCvmgogn-F=76^yQ#UovV`R0Y7zn})^^47M9LIj!q#;B@<3&fojgWpUo5z( zv^@A)K+q$XJz=C-foX`I#T*ZHdDtcf6JfAMpT2J!^~(XSCPIA*jAaxtv#`l7orFln zL7Xm3jy2bE>Ba7s2X>#e$`b02t7#Y1*NF*DW-`=CBG*ERue5+v-a66HIBxia<`z9m zs4sGY$u<_~L>C*ay*;CT8uUNT43@_cn*jB(o)#V%2yo6}u@!Y`10mvssh*D^uA&6$ z?s*h7xmVFmlJIH+#+uRE05@N9x`ynFH8@`Y^W?*6!9(yvxB(Ee}dwuRm zO7hWSDsCz<+9pDBg1&J~VBZF_6ZfrXqYJ9RhKO+M;IgVe0fx8Hw^l1lNItG6vb%)q zkNDVw_ubV;)TPVr!1dawdY4#_bLo?qN{^R6N$q$H5Q;@?l}a-}w<62bLq7;*-#5KH z-A*;h-DvQ(YzDlV4o4+6-UNojK^efQ83!04AB9;!J_DGB%)`KAr=(e)6|>Clr2qEH zQu%Ysns$yy12YNl78^n5u;7x5s^}5vBpf{=ffi5)d+yk2F`fjK9D<|ny&2UDNVODt za3WMEmSK$UiKJwLLvpI?LI;4tmq>&@{x@+#l-*LjURgr*aWx_JZv#@*zzj?XZ8NJa zxx_*o{n7PsCIkg#VYw?M6;F;ozKi%2ri93dvcAj@)%(omIRyfv~D3(trrYRYsc+ z3Bs3DwMHd)>;wppLW6u^grj9zCH~2I8_COJS)L!M0JrFz#r|oR>O!GDsnwQHer#2D zLV8gV`WlXRD&#j9io9ELsQF5$v*akgW4OhcNYJ)OwL2QTFQa{?D62&1-97C=un76f zeh`m3ne!+O&Ks$t(exrOB%6;DDrUI*By=~ZEfvt?s=BQtbPO*LB6b&SNnm&Zf8`zW zxuZnnF$^6{FibW)WYrU^BL4qG3C8Ifhb=>Sed@g#)z{O#NF@OBWT-AyMfM^@CdNjm zw2;6j8t&_=X9pm3aptpKa=f5=t+rIw+`6ioU;F9Qf9@8?Q8pw%j{R2S8_SSM3$fb0-Z8Z{Q1V#^qi*BoWcbgyJaV>XEFYmfyXgha~mdQekuJ zsxr+>5}5WD+)h1&*T9ek7i+>bl(7a~@oc=SZQqk^(g+uF-h{Lol80#Rz^J2*cV(nc zGtglSQHqv65z>pbmH6wp8nbmR>l;Z&rv(nGVh(Dimh#8$xE)w!lh%8}+k%}1%4GDrkKyvg<*643NCy2q?*nrCOT`2422;-B7-$(_llS8l_hi^S5r;xl?9v_?8ML!sm57a zV}-%C9K+d?)CoPjy=@mG3=7?t7e|$ePoTWPS;%&X;BTm-wN}RYg`6)%{-4MIFqc?_ zgarl|NjV0IZKOIkMAp%~TXKMte~6;jb?KazWbusI zo%{wiHa?^i)=gQQ|ECtN?&1IDUqgY00u2Qk3N#dGD9})#p+G}{h5`)*8VWQN_&HGE z^?k>)Q~ze7>8fUo-2JH^O}~Kfgbg4`C}YS!KOQT7q=A2v)KM5VhLeEtw%3-N`o~t4 zgpOWu`#~JP^Y;9W=wPg90RBgPixo-9f{m<-pci`R1qxuQTId@AS5bv(GFmUgk^e8V CWmOOW