Dirty mastodon rake task to migrate a database to pleroma
Created on:       Expiry on:
/1/lib/tasks/pleroma.rake 11 KB (text/x-c++)
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. class PleromaUsers < ActiveRecord::Base
  2. self.table_name = 'users'
  3. end
  4. class PleromaObjects < ActiveRecord::Base
  5. self.table_name = 'objects'
  6. end
  7. class PleromaActivities < ActiveRecord::Base
  8. self.table_name = 'activities'
  9. end
  10. # Migration methods
  11. # This one: would work
  12. # Replaying from mastodon to pleroma
  13. # 1. create accounts from SQL with the same method of this rake task
  14. # 2. for every status, follow, etc. we would POST /inbox with the AP object from the serializer
  15. # It's needed to have mastodon running on the "prod" domain, and pleroma on another port
  16. # Because pleroma will fetch everything it tries to ingest
  17. # Also need to be deactivated on pleroma the HTTP Signature check plug
  18. # First try, bare one, doesn't really seems to work well, statuses count seems ok but the timeline stay empty
  19. # bugs: (not tested at all with mastofe)
  20. # following count displays -1
  21. # follow and accept follow activities are generated, what may be missing ?
  22. # replying does work, but click on the timestamp to load conversation doesn't load the post responded to
  23. # conversations seems weird, the first imported status is a public one, others seems attached to it ?
  24. # favs doesn't seems to show
  25. # but boosting the toot which have been fav-imported fix it and it shows after reload as faved !
  26. # something may be wrong with the Activity or Object
  27. # Doesn't looks like I can access another user as /users/1 for ex. (imported two users, 1/ = admin, 2/ = me)
  28. # Object.data->>'id' is probably wrong everywhere ...
  29. # Ok, pleroma seems to use relationship by Id at some points like :
  30. # SELECT a0."id", a0."data", a0."local", a0."actor", a0."recipients", a0."inserted_at", a0."updated_at" FROM "activities" AS a0 WHERE (a0."id" = $1) [7]
  31. # When like deleting a favorite
  32. # This may be an issue if we have IDs not matching
  33. # Various notes from kaniini:
  34. # Object for all, User.info['following_count'] pre-calculations for -1 count, Replies, conversations and favs : probs mismatch between object and activity table (transmogrifier thingy)
  35. # working:
  36. # toots shows in pleromafe
  37. # Boost seems to works
  38. # Notes:
  39. # Activity.local is true when it's (from ?) the local instance
  40. # Best to do:
  41. # .inserted_at and .updated_at of object / activity must use the ones from the AP object ?
  42. MASTO_DB = :development
  43. PLERO_DB = :pleroma
  44. LOCAL_INSTANCE = "http://localhost:3000/"
  45. def migrate_users
  46. ActiveRecord::Base.establish_connection(MASTO_DB)
  47. # Use accounts since pleroma store local *and* remotes users in the same table, using User.local to differentiate them
  48. Account.all.each do |account|
  49. ActiveRecord::Base.establish_connection(MASTO_DB)
  50. user_info = ActiveModelSerializers::SerializableResource.new(account, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter).as_json
  51. user = account.user
  52. follow_count = account.following_count
  53. ActiveRecord::Base.establish_connection(PLERO_DB)
  54. # Create the pleroma user
  55. a = PleromaUsers.new
  56. pleroma_user_info = {}
  57. # handle specific fields for local and remote users
  58. if account.local?
  59. a.email = user.email
  60. pleroma_user_info[:is_moderator] = user.moderator
  61. end
  62. a.local = account.local?
  63. a.name = account.username
  64. a.nickname = account.username
  65. a.bio = account.note
  66. pleroma_user_info[:keys] = user_info[:publicKey][:publicKeyPem]
  67. pleroma_user_info[:following_count] = follow_count
  68. pleroma_user_info[:follower_count] = account.followers_count
  69. pleroma_user_info[:note_count] = account.statuses_count
  70. pleroma_user_info[:locked] = account.locked
  71. a.info = pleroma_user_info
  72. a.ap_id = user_info[:id]
  73. # avatar
  74. a.follower_address = user_info[:followers]
  75. a.following = user_info[:following]
  76. #a.password_hash = user.encrypted_password
  77. # Passwords are incompatible type, mastodon: bcrypt, pleroma: pbkfthing2
  78. a.inserted_at = user.created_at
  79. a.updated_at = user.updated_at
  80. a.save!
  81. end
  82. end
  83. def migrate_follows
  84. ActiveRecord::Base.establish_connection(MASTO_DB)
  85. Follow.all.each do |follow|
  86. ActiveRecord::Base.establish_connection(MASTO_DB)
  87. follow_object = ActiveModelSerializers::SerializableResource.new(follow, serializer: ActivityPub::FollowSerializer, adapter: ActivityPub::Adapter).as_json
  88. accept_follow_object = ActiveModelSerializers::SerializableResource.new(follow, serializer: ActivityPub::AcceptFollowSerializer, adapter: ActivityPub::Adapter).as_json
  89. ActiveRecord::Base.establish_connection(PLERO_DB)
  90. # Follow request object
  91. po = PleromaObjects.new
  92. po.data = {
  93. :cc => ["https://www.w3.org/ns/activitystreams#Public"],
  94. :id => follow_object[:object],
  95. :to => [follow_object[:object]],
  96. :type => "Follow",
  97. :actor => follow_object[:actor],
  98. :object => follow_object[:object],
  99. :published => follow.created_at
  100. }
  101. po.inserted_at = follow.created_at
  102. po.updated_at = follow.updated_at
  103. po.save!
  104. # Follow activity
  105. pa = PleromaActivities.new
  106. pa.data = follow_object
  107. pa.inserted_at = follow.created_at
  108. pa.updated_at = follow.updated_at
  109. pa.local = follow_object[:actor].include? LOCAL_INSTANCE
  110. pa.recipients = [follow_object[:object]]
  111. pa.actor = follow_object[:actor]
  112. pa.save!
  113. # Accept follow object
  114. poa = PleromaObjects.new
  115. poa.data = {
  116. :id => accept_follow_object[:object][:id],
  117. :to => [accept_follow_object[:object][:actor]],
  118. :type => "Follow",
  119. :object => accept_follow_object[:object],
  120. :actor => accept_follow_object[:actor]
  121. }
  122. poa.inserted_at = follow.created_at
  123. poa.updated_at = follow.updated_at
  124. poa.save!
  125. # Accept activity
  126. paf = PleromaActivities.new
  127. paf.data = accept_follow_object
  128. paf.inserted_at = follow.created_at
  129. paf.updated_at = follow.updated_at
  130. paf.local = accept_follow_object[:actor].include? LOCAL_INSTANCE
  131. paf.recipients = [accept_follow_object[:object][:actor]]
  132. paf.actor = accept_follow_object[:actor]
  133. paf.save!
  134. end
  135. end
  136. def migrate_posts
  137. ActiveRecord::Base.establish_connection(MASTO_DB)
  138. Status.all.each do |status|
  139. ActiveRecord::Base.establish_connection(MASTO_DB)
  140. status_ap = ActiveModelSerializers::SerializableResource.new(status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter).as_json
  141. ActiveRecord::Base.establish_connection(PLERO_DB)
  142. # For Notes, the Object.data is the content of the Activity.data.object
  143. # Create object
  144. po = PleromaObjects.new
  145. po.data = status_ap[:object]
  146. po.inserted_at = status.created_at
  147. po.updated_at = status.updated_at
  148. po.save!
  149. # Create Activity
  150. pa = PleromaActivities.new
  151. pa.data = status_ap
  152. pa.inserted_at = status.created_at
  153. pa.updated_at = status.updated_at
  154. pa.local = status_ap[:actor].include? LOCAL_INSTANCE
  155. if status_ap[:to].class == Array
  156. pa.recipients = status_ap[:to]
  157. else
  158. pa.recipients = [status_ap[:to]]
  159. end
  160. pa.actor = status_ap[:actor]
  161. pa.save!
  162. end
  163. end
  164. def migrate_rts
  165. ActiveRecord::Base.establish_connection(MASTO_DB)
  166. # It looks like in mastodon, boosts are differenciated by having Status.reblog_of_id
  167. # Pleroma uses announcments / announcement_count like Favourites
  168. Status.where.not(reblog_of_id: nil).each do |boost|
  169. ActiveRecord::Base.establish_connection(MASTO_DB)
  170. if not boost.reblog?
  171. next
  172. end
  173. status_ap = ActiveModelSerializers::SerializableResource.new(boost.reblog, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter).as_json
  174. ActiveRecord::Base.establish_connection(PLERO_DB)
  175. # Update object
  176. po = PleromaObjects.where("data->>'id' = ?", status_ap[:object][:id]).first
  177. if po.data.include?(:announcements)
  178. po.data[:announcements] << status_ap[:actor]
  179. else
  180. po.data[:announcements] = [status_ap[:actor]]
  181. end
  182. po.data[:announcement_count] = po.data[:announcements].length
  183. po.save
  184. end
  185. end
  186. def migrate_favs
  187. ActiveRecord::Base.establish_connection(MASTO_DB)
  188. Favourite.all.each do |fav|
  189. ActiveRecord::Base.establish_connection(MASTO_DB)
  190. status_ap = ActiveModelSerializers::SerializableResource.new(fav, serializer: ActivityPub::LikeSerializer, adapter: ActivityPub::Adapter).as_json
  191. ActiveRecord::Base.establish_connection(PLERO_DB)
  192. # Update object
  193. po = PleromaObjects.where("data->>'id' = ?", status_ap[:object]).first
  194. if po.data.include?(:likes)
  195. po.data[:likes] << status_ap[:actor]
  196. else
  197. po.data[:likes] = [status_ap[:actor]]
  198. end
  199. po.data[:like_count] = po.data[:likes].length
  200. po.save
  201. print(status_ap)
  202. # Update activity
  203. paa = PleromaActivities.where("data->'object'->>'id' = ?", status_ap[:object]).first
  204. if paa.data.include?(:likes)
  205. paa.data[:likes] << status_ap[:actor]
  206. else
  207. paa.data[:likes] = [status_ap[:actor]]
  208. end
  209. paa.data[:like_count] = paa.data[:likes].length
  210. paa.save
  211. # Create Activity
  212. pa = PleromaActivities.new
  213. pa.data = status_ap
  214. pa.inserted_at = fav.created_at
  215. pa.updated_at = fav.updated_at
  216. pa.local = status_ap[:actor].include? LOCAL_INSTANCE
  217. if status_ap[:object].class == Array
  218. pa.recipients = status_ap[:object]
  219. else
  220. pa.recipients = [status_ap[:object]]
  221. end
  222. pa.actor = status_ap[:actor]
  223. pa.save!
  224. end
  225. end
  226. def generate_rewritemap
  227. end
  228. namespace :pleroma do
  229. desc 'Migrate to pleroma'
  230. task migrate: :environment do
  231. print("### users")
  232. migrate_users
  233. print("### follows")
  234. migrate_follows
  235. print("### posts")
  236. migrate_posts
  237. print("### favs")
  238. migrate_favs
  239. print("### rts")
  240. migrate_rts
  241. print("### rewritemap")
  242. generate_rewritemap
  243. end
  244. end
/2/config/database.yml 1.2 KB (text/plain)
123456789101112131415161718192021222324252627282930313233343536373839
  1. default: &default
  2. adapter: postgresql
  3. pool: <%= ENV["DB_POOL"] || ENV['MAX_THREADS'] || 5 %>
  4. timeout: 5000
  5. encoding: unicode
  6. development:
  7. <<: *default
  8. database: <%= ENV['DB_NAME'] || 'mastodon_development' %>
  9. username: <%= ENV['DB_USER'] %>
  10. password: <%= ENV['DB_PASS'] %>
  11. host: <%= ENV['DB_HOST'] %>
  12. port: <%= ENV['DB_PORT'] %>
  13. # Warning: The database defined as "test" will be erased and
  14. # re-generated from your development database when you run "rake".
  15. # Do not set this db to the same as development or production.
  16. test:
  17. <<: *default
  18. database: <%= ENV['DB_NAME'] || 'mastodon' %>_test<%= ENV['TEST_ENV_NUMBER'] %>
  19. username: <%= ENV['DB_USER'] %>
  20. password: <%= ENV['DB_PASS'] %>
  21. host: <%= ENV['DB_HOST'] %>
  22. port: <%= ENV['DB_PORT'] %>
  23. production:
  24. <<: *default
  25. database: <%= ENV['DB_NAME'] || 'mastodon_production' %>
  26. username: <%= ENV['DB_USER'] || 'mastodon' %>
  27. password: <%= ENV['DB_PASS'] || '' %>
  28. host: <%= ENV['DB_HOST'] || 'localhost' %>
  29. port: <%= ENV['DB_PORT'] || 5432 %>
  30. prepared_statements: <%= ENV['PREPARED_STATEMENTS'] || 'true' %>
  31. pleroma:
  32. <<: *default
  33. database: 'pleroma_devmasto'
  34. username: 'dashie'
  35. password: 'dashie'
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127
  1. require 'net/http'
  2. require 'uri'
  3. require 'json'
  4. class PleromaUsers < ActiveRecord::Base
  5. self.table_name = 'users'
  6. end
  7. # bugs:
  8. # Object.data->>'id' is probably wrong everywhere ...
  9. # Various notes from kaniini:
  10. # Object for all, User.info['following_count'] pre-calculations for -1 count, Replies, conversations and favs : probs mismatch between object and activity table (transmogrifier thingy)
  11. # Notes:
  12. # Activity.local is true when it's (from ?) the local instance
  13. # Best to do:
  14. # .inserted_at and .updated_at of object / activity must use the ones from the AP object ?
  15. MASTO_DB = :development
  16. PLERO_DB = :pleroma
  17. LOCAL_INSTANCE = "http://0.0.0.0:3000"
  18. def migrate_users
  19. ActiveRecord::Base.establish_connection(MASTO_DB)
  20. # Use accounts since pleroma store local *and* remotes users in the same table, using User.local to differentiate them
  21. Account.all.each do |account|
  22. ActiveRecord::Base.establish_connection(MASTO_DB)
  23. user_info = ActiveModelSerializers::SerializableResource.new(account, serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter).as_json
  24. user = account.user
  25. follow_count = account.following_count
  26. ActiveRecord::Base.establish_connection(PLERO_DB)
  27. # Create the pleroma user
  28. a = PleromaUsers.new
  29. pleroma_user_info = {}
  30. # handle specific fields for local and remote users
  31. if account.local?
  32. a.email = user.email
  33. pleroma_user_info[:is_moderator] = user.moderator
  34. end
  35. a.local = account.local?
  36. a.name = account.username
  37. a.nickname = account.username
  38. a.bio = account.note
  39. pleroma_user_info[:keys] = user_info[:publicKey][:publicKeyPem]
  40. pleroma_user_info[:following_count] = follow_count
  41. pleroma_user_info[:follower_count] = account.followers_count
  42. pleroma_user_info[:note_count] = account.statuses_count
  43. pleroma_user_info[:locked] = account.locked
  44. a.info = pleroma_user_info
  45. a.ap_id = user_info[:id]
  46. # avatar
  47. a.follower_address = user_info[:followers]
  48. a.following = user_info[:following]
  49. #a.password_hash = user.encrypted_password
  50. # Passwords are incompatible type, mastodon: bcrypt, pleroma: pbkfthing2
  51. a.inserted_at = user.created_at
  52. a.updated_at = user.updated_at
  53. a.save!
  54. end
  55. end
  56. def replay_follows
  57. ActiveRecord::Base.establish_connection(MASTO_DB)
  58. Follow.all.each do |follow|
  59. ActiveRecord::Base.establish_connection(MASTO_DB)
  60. follow_object = ActiveModelSerializers::SerializableResource.new(follow, serializer: ActivityPub::FollowSerializer, adapter: ActivityPub::Adapter).as_json
  61. accept_follow_object = ActiveModelSerializers::SerializableResource.new(follow, serializer: ActivityPub::AcceptFollowSerializer, adapter: ActivityPub::Adapter).as_json
  62. end
  63. end
  64. def replay_statuses
  65. ActiveRecord::Base.establish_connection(MASTO_DB)
  66. Status.all.each do |status|
  67. ActiveRecord::Base.establish_connection(MASTO_DB)
  68. status_ap = ActiveModelSerializers::SerializableResource.new(status, serializer: ActivityPub::ActivitySerializer, adapter: ActivityPub::Adapter).as_json
  69. uri = URI.parse("#{LOCAL_INSTANCE}/inbox")
  70. #uri = URI.parse("#{status_ap[:actor]}/inbox")
  71. header = {'Content-Type': 'application/activity+json'}
  72. http = Net::HTTP.new(uri.host, uri.port)
  73. request = Net::HTTP::Post.new(uri.request_uri, header)
  74. request.body = status_ap.to_json
  75. response = http.request(request)
  76. end
  77. end
  78. def replay_favs
  79. end
  80. def replay_everything
  81. replay_follows
  82. replay_statuses
  83. replay_favs
  84. end
  85. def generate_rewritemap
  86. end
  87. namespace :pleroma do
  88. desc 'Migrate users'
  89. task step1_migrate_users: :environment do
  90. migrate_users
  91. end
  92. desc 'Replay everything from AP'
  93. task step2_replay_everything: :environment do
  94. replay_everything
  95. end
  96. desc 'Generate various rewritemap'
  97. task step99_rewritemap: :environment do
  98. generate_rewritemap
  99. end
  100. end
© 2017-2018 git.txt - version: 0.5.2 - b28934a7ddecd31f2e40cb7058fa1fb5e9a1acc8 - page: 373ms - template: 3ms
Gitxts: 42, managed: 86
Sources - Go1.11.5