diff --git a/.gitignore b/.gitignore index 59003f5eabd..98af4c68ea5 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,5 @@ go.work.sum # Token deployment cache files tests-functional/snt_addresses.json* +# Codex AI (chatGPT) +.codex diff --git a/.vscode/settings.json b/.vscode/settings.json index 37794072037..c8ebe0dfe26 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,14 +1,22 @@ { "go.testTags": "gowaku_skip_migrations,gowaku_no_rln", - "cSpell.words": [ - "unmarshalling" - ], + "cSpell.words": ["unmarshalling"], "gopls": { - "formatting.local": "github.com/status-im/status-go" + "formatting.local": "github.com/status-im/status-go", + "build.buildFlags": ["-tags=gowaku_skip_migrations,gowaku_no_rln"] }, // format all files on save if a formatter is available "editor.formatOnSave": true, - // I use "goimports" instead of "gofmt" + // use "goimports" instead of "gofmt" // because it does the same thing but also formats imports "go.formatTool": "goimports", + "go.toolsEnvVars": { + "CGO_ENABLED": "1" + }, + "go.testEnvVars": { + "CGO_CFLAGS": "-I${workspaceFolder}/../nim-sds/library", + "CGO_LDFLAGS": "-L${workspaceFolder}/../nim-sds/build -lsds -Wl,-rpath,${workspaceFolder}/../nim-sds/build" + }, + "go.buildFlags": ["-tags=gowaku_skip_migrations,gowaku_no_rln"], + "go.testFlags": ["-tags=gowaku_skip_migrations,gowaku_no_rln"] } diff --git a/README.md b/README.md index fe55d4c5e5a..49cb768633a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ A comprehensive list of `status-go` functionality can be found in [Brief overvie # Docs -- [How to Build](./building.md) +- [How to Build](docs/building.md) - [How to Contribute](CONTRIBUTING.md) - [How to Release](docs/RELEASING.md) - [How to run status-go as HTTP server](/cmd/status-backend/README.md) diff --git a/internal/db/appdatabase/migrations/sql/1771180800_rename_magnetlink_to_archive_link.up.sql b/internal/db/appdatabase/migrations/sql/1771180800_rename_magnetlink_to_archive_link.up.sql new file mode 100644 index 00000000000..5786e55d47d --- /dev/null +++ b/internal/db/appdatabase/migrations/sql/1771180800_rename_magnetlink_to_archive_link.up.sql @@ -0,0 +1,3 @@ +-- Rename magnetlink columns to archive_link for clarity +ALTER TABLE communities_archive_info RENAME COLUMN magnetlink_clock TO archive_link_clock; +ALTER TABLE communities_archive_info RENAME COLUMN last_magnetlink_uri TO last_archive_link; diff --git a/protocol/communities/archive/archive_manager.go b/protocol/communities/archive/archive_manager.go new file mode 100644 index 00000000000..8aa758133a9 --- /dev/null +++ b/protocol/communities/archive/archive_manager.go @@ -0,0 +1,424 @@ +//go:build !disable_history_archives + +// // +build !disable_history_archives +package archive + +import ( + "context" + "crypto/ecdsa" + "errors" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/status-im/status-go/common" + cryptotypes "github.com/status-im/status-go/internal/crypto/types" + "github.com/status-im/status-go/pkg/messaging" + messagingtypes "github.com/status-im/status-go/pkg/messaging/types" + archivetorrent "github.com/status-im/status-go/protocol/communities/archive/torrent" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" + archiveutils "github.com/status-im/status-go/protocol/communities/archive/utils" + "github.com/status-im/status-go/protocol/protobuf" + "github.com/status-im/status-go/signal" +) + +type ArchiveManager struct { + historyArchiveDownloadTasks map[string]*archivetypes.HistoryArchiveDownloadTask + downloadTasksMu sync.RWMutex // protects historyArchiveDownloadTasks + historyArchiveTasksWaitGroup sync.WaitGroup + historyArchiveTasks sync.Map // stores `chan struct{}` + + logger *zap.Logger + persistence archivetypes.PersistenceProvider + messaging *messaging.API + identity *ecdsa.PrivateKey + + publisher archivetypes.HistoryArchivePublisher + backend ArchiveServiceBackend +} + +// NewArchiveManager this function is only built and called when the "disable_history_archives" build tag is not set. +// In this case this version of NewArchiveManager will return a fully functional ArchiveManager ensuring that the +// build command will import and build the archive dependencies. +// NOTE: It is intentional that this file contains the identical function name as in "archive_manager_nop.go" +// +// This function implements the NOP pattern: it ALWAYS returns a valid ArchiveService instance, +// never nil. When archive functionality is not enabled (via config.Enabled field), it returns +// the ArchiveManagerNop which safely does nothing for all operations. +// This eliminates the need for nil/Enabled checks throughout the codebase. +func NewArchiveManager(amc *archivetypes.ArchiveManagerConfig) ArchiveService { + // Depending on which config is provided AND enabled, we instantiate the corresponding + // concrete ArchiveManager backend implementation. + var backend ArchiveServiceBackend + + if amc.TorrentConfig != nil && amc.TorrentConfig.Enabled { + // Torrent-based archive backend + backend = archivetorrent.NewArchiveManagerTorrent( + amc.TorrentConfig, + amc.Logger, + amc.Persistence, + amc.Messaging, + amc.Identity, + amc.Publisher, + ) + } else { + // No enabled configuration - return the NOP implementation + // This ensures we always return a valid instance that safely does nothing + return &ArchiveManagerNop{} + } + + return &ArchiveManager{ + historyArchiveDownloadTasks: make(map[string]*archivetypes.HistoryArchiveDownloadTask), + + logger: amc.Logger, + persistence: amc.Persistence, + messaging: amc.Messaging, + identity: amc.Identity, + + publisher: amc.Publisher, + backend: backend, + } +} + +// ArchiveServiceBackend interface implementation - delegates to backend + +func (m *ArchiveManager) SetOnline(online bool) { + m.backend.SetOnline(online) +} + +func (m *ArchiveManager) Start() error { + return m.backend.Start() +} + +func (m *ArchiveManager) Stop() error { + // stopHistoryArchiveTasksIntervals should be called unconditionally + m.stopHistoryArchiveTasksIntervals() + return m.backend.Stop() +} + +func (m *ArchiveManager) IsStarted() bool { + return m.backend.IsStarted() +} + +func (m *ArchiveManager) SeedHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) error { + return m.backend.SeedHistoryArchive(communityID, archiveLink) +} + +func (m *ArchiveManager) UnseedHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) { + m.backend.UnseedHistoryArchive(communityID, archiveLink) +} + +func (m *ArchiveManager) IsSeedingHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) bool { + return m.backend.IsSeedingHistoryArchive(communityID, archiveLink) +} + +func (m *ArchiveManager) DownloadHistoryArchives(communityID cryptotypes.HexBytes, archiveLink string, cancelTask chan struct{}) (*archivetypes.HistoryArchiveDownloadTaskInfo, error) { + return m.backend.DownloadHistoryArchives(communityID, archiveLink, cancelTask) +} + +func (m *ArchiveManager) CreateHistoryArchiveFromMessages(communityID cryptotypes.HexBytes, messages []*messagingtypes.ReceivedMessage, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { + return m.backend.CreateHistoryArchiveFromMessages(communityID, messages, topics, startDate, endDate, partition, encrypt) +} + +func (m *ArchiveManager) CreateHistoryArchiveFromDB(communityID cryptotypes.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { + return m.backend.CreateHistoryArchiveFromDB(communityID, topics, startDate, endDate, partition, encrypt) +} + +func (m *ArchiveManager) LoadArchiveMessages(ctx context.Context, communityID cryptotypes.HexBytes, archiveLink string, downloadedArchiveID string) ([]*protobuf.WakuMessage, error) { + return m.backend.LoadArchiveMessages(ctx, communityID, archiveLink, downloadedArchiveID) +} + +// ArchiveService interface implementation - storage-agnostic operations + +func (m *ArchiveManager) GetHistoryArchiveLink(communityID cryptotypes.HexBytes) (string, error) { + return m.persistence.GetLastSeenArchiveLink(communityID) +} + +func (m *ArchiveManager) GetCommunityChatsFilters(communityID cryptotypes.HexBytes) (messagingtypes.ChatFilters, error) { + chatIDs, err := m.persistence.GetCommunityChatIDs(communityID) + if err != nil { + return nil, err + } + + filters := messagingtypes.ChatFilters{} + for _, cid := range chatIDs { + filter := m.messaging.ChatFilterByChatID(cid) + if filter != nil { + filters = append(filters, filter) + } + } + return filters, nil +} + +func (m *ArchiveManager) GetCommunityChatsTopics(communityID cryptotypes.HexBytes) ([]messagingtypes.ContentTopic, error) { + filters, err := m.GetCommunityChatsFilters(communityID) + if err != nil { + return nil, err + } + + topics := []messagingtypes.ContentTopic{} + for _, filter := range filters { + topics = append(topics, filter.ContentTopic()) + } + + return topics, nil +} + +func (m *ArchiveManager) GetHistoryArchivePartitionStartTimestamp(communityID cryptotypes.HexBytes) (uint64, error) { + exists, err := m.persistence.CommunityExists(&m.identity.PublicKey, communityID) + if err != nil { + m.logger.Error("failed to check community existence", zap.Error(err)) + return 0, err + } + + if !exists { + m.logger.Error("community not found for this id") + return 0, errors.New("community not found") + } + + filters, err := m.GetCommunityChatsFilters(communityID) + if err != nil { + m.logger.Error("failed to get community chats filters", zap.Error(err)) + return 0, err + } + + universalChatID := archiveutils.UniversalChatIDFromCommunityID(communityID) + + if m.messaging != nil { + filter := m.messaging.ChatFilterByChatID(universalChatID) + if filter != nil { + filters = append(filters, filter) + } + } + + if len(filters) == 0 { + // If we don't have chat filters, we likely don't have any chats + // associated to this community, which means there's nothing more + // to do here + return 0, nil + } + + topics := []messagingtypes.ContentTopic{} + for _, filter := range filters { + topics = append(topics, filter.ContentTopic()) + } + + lastArchiveEndDateTimestamp, err := m.getLastMessageArchiveEndDate(communityID) + if err != nil { + m.logger.Error("failed to get last archive end date", zap.Error(err)) + return 0, err + } + + if lastArchiveEndDateTimestamp == 0 { + // If we don't have a tracked last message archive end date, it + // means we haven't created an archive before, which means + // the next thing to look at is the oldest waku message timestamp for + // this community + lastArchiveEndDateTimestamp, err = m.getOldestWakuMessageTimestamp(topics) + if err != nil { + m.logger.Error("failed to get oldest waku message timestamp", zap.Error(err)) + return 0, err + } + if lastArchiveEndDateTimestamp == 0 { + // This means there's no waku message stored for this community so far + // (even after requesting possibly missed messages), so no messages exist yet that can be archived + m.logger.Debug("can't find valid `lastArchiveEndTimestamp`") + return 0, nil + } + } + + return lastArchiveEndDateTimestamp, nil +} + +func (m *ArchiveManager) CreateAndSeedHistoryArchive(communityID cryptotypes.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) error { + err := m.backend.CreateAndSeedHistoryArchive(communityID, topics, startDate, endDate, partition, encrypt) + + if err != nil { + m.logger.Error("failed to create and seed history archive", zap.Error(err)) + return err + } + + // one way of publishing index succeeded - we can publish the seeding signal + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + HistoryArchivesSeedingSignal: &signal.HistoryArchivesSeedingSignal{ + CommunityID: communityID.String(), + }, + }) + + return nil +} + +func (m *ArchiveManager) StartHistoryArchiveTasksInterval(communityID cryptotypes.HexBytes, chatID string, encrypted bool, interval time.Duration) { + defer common.LogOnPanic() + id := cryptotypes.EncodeHex(communityID) + + if _, exists := m.historyArchiveTasks.Load(id); exists { + m.logger.Error("history archive tasks interval already in progress", zap.String("id", id)) + return + } + + cancel := make(chan struct{}) + m.historyArchiveTasks.Store(id, cancel) + m.historyArchiveTasksWaitGroup.Add(1) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + m.logger.Debug("starting history archive tasks interval", zap.String("id", id)) + for { + select { + case <-ticker.C: + m.logger.Debug("starting archive task...", zap.String("id", id)) + + lastArchiveEndDateTimestamp, err := m.GetHistoryArchivePartitionStartTimestamp(communityID) + if err != nil { + m.logger.Error("failed to get last archive end date", zap.Error(err)) + continue + } + + if lastArchiveEndDateTimestamp == 0 { + // This means there are no waku messages for this community, + // so nothing to do here + m.logger.Debug("couldn't determine archive start date - skipping") + continue + } + + topics, err := m.GetCommunityChatsTopics(communityID) + if err != nil { + m.logger.Error("failed to get community chat topics ", zap.Error(err)) + continue + } + filter := m.messaging.ChatFilterByChatID(chatID) + if filter == nil { + m.logger.Error("failed to get chat filter", zap.String("community's UniversalChatID", chatID)) + continue + } + // adding the content-topic used for member updates. + // since member updates would not be too frequent i.e only addition/deletion would add a new message, + // this shouldn't cause too much increase in size of archive generated. + topics = append(topics, filter.ContentTopic()) + + ts := time.Now().Unix() + to := time.Unix(ts, 0) + lastArchiveEndDate := time.Unix(int64(lastArchiveEndDateTimestamp), 0) + + err = m.CreateAndSeedHistoryArchive(communityID, topics, lastArchiveEndDate, to, interval, encrypted) + if err != nil { + m.logger.Error("failed to create and seed history archive", zap.Error(err)) + continue + } + case <-cancel: + lastSeenArchiveLink, err := m.persistence.GetLastSeenArchiveLink(communityID) + if err != nil { + m.logger.Debug("[LogosStorage][start_history_archive_tasks_interval] failed to get last seen archive link - proceeding without un-seeding", zap.Error(err)) + } else { + m.UnseedHistoryArchive(communityID, lastSeenArchiveLink) + } + m.historyArchiveTasks.Delete(id) + m.historyArchiveTasksWaitGroup.Done() + return + } + } +} + +func (m *ArchiveManager) StopHistoryArchiveTasksInterval(communityID cryptotypes.HexBytes) { + task, exists := m.historyArchiveTasks.Load(communityID.String()) + if exists { + m.logger.Info("Stopping history archive tasks interval", zap.Any("id", communityID.String())) + close(task.(chan struct{})) // Need to cast to the chan + } +} + +func (m *ArchiveManager) GetHistoryArchiveDownloadTask(communityID string) *archivetypes.HistoryArchiveDownloadTask { + m.downloadTasksMu.RLock() + defer m.downloadTasksMu.RUnlock() + return m.historyArchiveDownloadTasks[communityID] +} + +func (m *ArchiveManager) AddHistoryArchiveDownloadTask(communityID string, task *archivetypes.HistoryArchiveDownloadTask) { + m.downloadTasksMu.Lock() + defer m.downloadTasksMu.Unlock() + m.historyArchiveDownloadTasks[communityID] = task +} + +func (m *ArchiveManager) RemoveHistoryArchiveDownloadTask(communityID string) { + m.downloadTasksMu.Lock() + defer m.downloadTasksMu.Unlock() + delete(m.historyArchiveDownloadTasks, communityID) +} + +func (m *ArchiveManager) PublishHistoryArchivesSeedingSignal(communityID cryptotypes.HexBytes) { + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + HistoryArchivesSeedingSignal: &signal.HistoryArchivesSeedingSignal{ + CommunityID: communityID.String(), + }, + }) +} + +func (m *ArchiveManager) GetDownloadedMessageArchiveIDs(communityID cryptotypes.HexBytes) ([]string, error) { + return m.persistence.GetDownloadedMessageArchiveIDs(communityID) +} + +func (m *ArchiveManager) SaveMessageArchiveID(communityID cryptotypes.HexBytes, hash string) error { + return m.persistence.SaveMessageArchiveID(communityID, hash) +} + +func (m *ArchiveManager) GetMessageArchiveIDsToImport(communityID cryptotypes.HexBytes) ([]string, error) { + return m.persistence.GetMessageArchiveIDsToImport(communityID) +} + +func (m *ArchiveManager) SetMessageArchiveIDImported(communityID cryptotypes.HexBytes, hash string, imported bool) error { + return m.persistence.SetMessageArchiveIDImported(communityID, hash, imported) +} + +func (m *ArchiveManager) GetHistoryTasksCount() int { + // sync.Map doesn't have a Len function, so we need to count manually + count := 0 + m.historyArchiveTasks.Range(func(_, _ interface{}) bool { + count++ + return true + }) + return count +} + +// private methods +func (m *ArchiveManager) stopHistoryArchiveTasksIntervals() { + m.historyArchiveTasks.Range(func(_, task interface{}) bool { + close(task.(chan struct{})) // Need to cast to the chan + return true + }) + // Stoping archive interval tasks is async, so we need + // to wait for all of them to be closed before we shutdown + // the torrent client + m.historyArchiveTasksWaitGroup.Wait() +} + +func (m *ArchiveManager) getOldestWakuMessageTimestamp(topics []messagingtypes.ContentTopic) (uint64, error) { + return m.persistence.GetOldestWakuMessageTimestamp(topics) +} + +func (m *ArchiveManager) getLastMessageArchiveEndDate(communityID cryptotypes.HexBytes) (uint64, error) { + return m.persistence.GetLastMessageArchiveEndDate(communityID) +} + +// Special functions +// These functions are not part of the ArchiveService interface. +// Some legacy tests are accessing implementation details and for this reason +// we need to expose these special accessors. + +// GetTorrentBackend returns the Torrent backend if available, for test purposes +func (m *ArchiveManager) GetTorrentBackend() (*archivetorrent.ArchiveManagerTorrent, error) { + if torrentBackend, ok := m.backend.(*archivetorrent.ArchiveManagerTorrent); ok { + return torrentBackend, nil + } + return nil, errors.New("backend is not ArchiveManagerTorrent") +} + +func (m *ArchiveManager) Wait() { + m.historyArchiveTasksWaitGroup.Wait() +} + +func (m *ArchiveManager) StopHistoryArchiveTasksIntervalsAndWait() { + m.stopHistoryArchiveTasksIntervals() +} diff --git a/protocol/communities/archive/archive_manager_nop.go b/protocol/communities/archive/archive_manager_nop.go new file mode 100644 index 00000000000..18907a686a5 --- /dev/null +++ b/protocol/communities/archive/archive_manager_nop.go @@ -0,0 +1,16 @@ +//go:build disable_history_archives +// +build disable_history_archives + +package archive + +import ( + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" +) + +// NewArchiveManager this function is only built and called when the "disable_history_archives" build tag is set. +// In this case this version of NewArchiveManager will return the NOP implementation ensuring that the +// build command will not import or build the archive dependencies for mobile builds. +// NOTE: It is intentional that this file contains the identical function name as in "archive_manager.go" +func NewArchiveManager(amc *archivetypes.ArchiveManagerConfig) ArchiveService { + return &ArchiveManagerNop{} +} diff --git a/protocol/communities/archive/archive_manager_nop_type.go b/protocol/communities/archive/archive_manager_nop_type.go new file mode 100644 index 00000000000..ea2a14acc7a --- /dev/null +++ b/protocol/communities/archive/archive_manager_nop_type.go @@ -0,0 +1,120 @@ +package archive + +import ( + "context" + "time" + + cryptotypes "github.com/status-im/status-go/internal/crypto/types" + messagingtypes "github.com/status-im/status-go/pkg/messaging/types" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" + "github.com/status-im/status-go/protocol/protobuf" +) + +// ArchiveManagerNop is a no-op implementation of ArchiveService. +// This type is always compiled (no build tags) so it can be used +// both when history archives are disabled at compile time, and when +// they're enabled but no configuration is provided at runtime. +type ArchiveManagerNop struct{} + +// ArchiveServiceBackend interface implementation + +func (m *ArchiveManagerNop) SetOnline(online bool) {} + +func (m *ArchiveManagerNop) Start() error { + return nil +} + +func (m *ArchiveManagerNop) Stop() error { + return nil +} + +func (m *ArchiveManagerNop) IsStarted() bool { + return false +} + +func (m *ArchiveManagerNop) SeedHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) error { + return nil +} + +func (m *ArchiveManagerNop) UnseedHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) { +} + +func (m *ArchiveManagerNop) IsSeedingHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) bool { + return false +} + +func (m *ArchiveManagerNop) DownloadHistoryArchives(communityID cryptotypes.HexBytes, archiveLink string, cancelTask chan struct{}) (*archivetypes.HistoryArchiveDownloadTaskInfo, error) { + return nil, nil +} + +func (m *ArchiveManagerNop) CreateHistoryArchiveFromMessages(communityID cryptotypes.HexBytes, messages []*messagingtypes.ReceivedMessage, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { + return nil, nil +} + +func (m *ArchiveManagerNop) CreateHistoryArchiveFromDB(communityID cryptotypes.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { + return nil, nil +} + +func (m *ArchiveManagerNop) LoadArchiveMessages(ctx context.Context, communityID cryptotypes.HexBytes, archiveLink string, downloadedArchiveID string) ([]*protobuf.WakuMessage, error) { + return nil, nil +} + +// ArchiveService interface implementation + +func (m *ArchiveManagerNop) GetHistoryArchiveLink(communityID cryptotypes.HexBytes) (string, error) { + return "", nil +} + +func (m *ArchiveManagerNop) GetCommunityChatsFilters(communityID cryptotypes.HexBytes) (messagingtypes.ChatFilters, error) { + return nil, nil +} + +func (m *ArchiveManagerNop) GetCommunityChatsTopics(communityID cryptotypes.HexBytes) ([]messagingtypes.ContentTopic, error) { + return nil, nil +} + +func (m *ArchiveManagerNop) GetHistoryArchivePartitionStartTimestamp(communityID cryptotypes.HexBytes) (uint64, error) { + return 0, nil +} + +func (m *ArchiveManagerNop) CreateAndSeedHistoryArchive(communityID cryptotypes.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) error { + return nil +} + +func (m *ArchiveManagerNop) StartHistoryArchiveTasksInterval(communityID cryptotypes.HexBytes, chatID string, encrypted bool, interval time.Duration) { +} + +func (m *ArchiveManagerNop) StopHistoryArchiveTasksInterval(communityID cryptotypes.HexBytes) {} + +func (m *ArchiveManagerNop) GetHistoryArchiveDownloadTask(communityID string) *archivetypes.HistoryArchiveDownloadTask { + return nil +} + +func (m *ArchiveManagerNop) AddHistoryArchiveDownloadTask(communityID string, task *archivetypes.HistoryArchiveDownloadTask) { +} + +func (m *ArchiveManagerNop) RemoveHistoryArchiveDownloadTask(communityID string) { +} + +func (m *ArchiveManagerNop) PublishHistoryArchivesSeedingSignal(communityID cryptotypes.HexBytes) { +} + +func (m *ArchiveManagerNop) GetDownloadedMessageArchiveIDs(communityID cryptotypes.HexBytes) ([]string, error) { + return nil, nil +} + +func (m *ArchiveManagerNop) SaveMessageArchiveID(communityID cryptotypes.HexBytes, hash string) error { + return nil +} + +func (m *ArchiveManagerNop) GetMessageArchiveIDsToImport(communityID cryptotypes.HexBytes) ([]string, error) { + return nil, nil +} + +func (m *ArchiveManagerNop) SetMessageArchiveIDImported(communityID cryptotypes.HexBytes, hash string, imported bool) error { + return nil +} + +func (m *ArchiveManagerNop) GetHistoryTasksCount() int { + return 0 +} diff --git a/protocol/communities/archive/archive_service.go b/protocol/communities/archive/archive_service.go new file mode 100644 index 00000000000..c103afef7a8 --- /dev/null +++ b/protocol/communities/archive/archive_service.go @@ -0,0 +1,51 @@ +package archive + +//go:generate go tool mockgen -package=mock_archive -source=archive_service.go -destination=mock/archive/archive_service.go + +import ( + "context" + "time" + + "github.com/status-im/status-go/internal/crypto/types" + messagingtypes "github.com/status-im/status-go/pkg/messaging/types" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" + "github.com/status-im/status-go/protocol/protobuf" +) + +type ArchiveServiceBackend interface { + SetOnline(bool) + Start() error + Stop() error + IsStarted() bool + SeedHistoryArchive(communityID types.HexBytes, archiveLink string) error + UnseedHistoryArchive(communityID types.HexBytes, archiveLink string) + IsSeedingHistoryArchive(communityID types.HexBytes, archiveLink string) bool + DownloadHistoryArchives(communityID types.HexBytes, archiveLink string, cancelTask chan struct{}) (*archivetypes.HistoryArchiveDownloadTaskInfo, error) + CreateHistoryArchiveFromMessages(communityID types.HexBytes, messages []*messagingtypes.ReceivedMessage, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) + CreateHistoryArchiveFromDB(communityID types.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) + CreateAndSeedHistoryArchive(communityID types.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) error + LoadArchiveMessages(ctx context.Context, communityID types.HexBytes, archiveLink string, downloadedArchiveID string) ([]*protobuf.WakuMessage, error) +} + +type ArchiveService interface { + // ArchiveServiceBackend provides a proxy interface to the underlying storage-specific + // implementation of the archive service. + ArchiveServiceBackend + // Storage-agnostic operations + GetHistoryArchiveLink(communityID types.HexBytes) (string, error) + GetCommunityChatsFilters(communityID types.HexBytes) (messagingtypes.ChatFilters, error) + GetCommunityChatsTopics(communityID types.HexBytes) ([]messagingtypes.ContentTopic, error) + GetHistoryArchivePartitionStartTimestamp(communityID types.HexBytes) (uint64, error) + CreateAndSeedHistoryArchive(communityID types.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) error + StartHistoryArchiveTasksInterval(communityID types.HexBytes, chatID string, encrypted bool, interval time.Duration) + StopHistoryArchiveTasksInterval(communityID types.HexBytes) + GetHistoryArchiveDownloadTask(communityID string) *archivetypes.HistoryArchiveDownloadTask + AddHistoryArchiveDownloadTask(communityID string, task *archivetypes.HistoryArchiveDownloadTask) + RemoveHistoryArchiveDownloadTask(communityID string) + PublishHistoryArchivesSeedingSignal(communityID types.HexBytes) + GetDownloadedMessageArchiveIDs(communityID types.HexBytes) ([]string, error) + SaveMessageArchiveID(communityID types.HexBytes, hash string) error + GetMessageArchiveIDsToImport(communityID types.HexBytes) ([]string, error) + SetMessageArchiveIDImported(communityID types.HexBytes, hash string, imported bool) error + GetHistoryTasksCount() int +} diff --git a/protocol/communities/archive/commons/archive_commons.go b/protocol/communities/archive/commons/archive_commons.go new file mode 100644 index 00000000000..3e33df2a730 --- /dev/null +++ b/protocol/communities/archive/commons/archive_commons.go @@ -0,0 +1,5 @@ +package commons + +import "errors" + +var ErrArchiveTimedout = errors.New("archive has timed out") diff --git a/protocol/communities/archive/consts/archive_consts.go b/protocol/communities/archive/consts/archive_consts.go new file mode 100644 index 00000000000..c1c522c93c2 --- /dev/null +++ b/protocol/communities/archive/consts/archive_consts.go @@ -0,0 +1,3 @@ +package consts + +const MaxArchiveSizeInBytes = 30000000 diff --git a/protocol/communities/archive/torrent/archive_manager_torrent.go b/protocol/communities/archive/torrent/archive_manager_torrent.go new file mode 100644 index 00000000000..4e82f48d622 --- /dev/null +++ b/protocol/communities/archive/torrent/archive_manager_torrent.go @@ -0,0 +1,1032 @@ +//go:build !disable_history_archives +// +build !disable_history_archives + +package torrent + +import ( + "context" + "crypto/ecdsa" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sort" + "sync" + "time" + + "github.com/anacrolix/torrent" + "github.com/anacrolix/torrent/bencode" + "github.com/anacrolix/torrent/metainfo" + + "google.golang.org/protobuf/proto" + + "github.com/status-im/status-go/signal" + + "go.uber.org/zap" + + "github.com/status-im/status-go/internal/crypto" + + cryptotypes "github.com/status-im/status-go/internal/crypto/types" + "github.com/status-im/status-go/params" + "github.com/status-im/status-go/pkg/messaging" + messagingtypes "github.com/status-im/status-go/pkg/messaging/types" + archivecommons "github.com/status-im/status-go/protocol/communities/archive/commons" + archiveconsts "github.com/status-im/status-go/protocol/communities/archive/consts" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" + archiveutils "github.com/status-im/status-go/protocol/communities/archive/utils" + "github.com/status-im/status-go/protocol/protobuf" +) + +type archiveMDSlice []*archiveMetadata + +type archiveMetadata struct { + hash string + from uint64 +} + +func (md archiveMDSlice) Len() int { + return len(md) +} + +func (md archiveMDSlice) Swap(i, j int) { + md[i], md[j] = md[j], md[i] +} + +func (md archiveMDSlice) Less(i, j int) bool { + return md[i].from > md[j].from +} + +type EncodedArchiveData struct { + padding int + bytes []byte +} + +type ArchiveManagerTorrent struct { + torrentConfig *params.TorrentConfig + torrentClient *torrent.Client + torrentTasks map[string]metainfo.Hash + torrentMu sync.RWMutex + + logger *zap.Logger + persistence archivetypes.PersistenceProvider + messaging *messaging.API + identity *ecdsa.PrivateKey + + publisher archivetypes.HistoryArchivePublisher +} + +var defaultAnnounceList = [][]string{ + {"udp://tracker.opentrackr.org:1337/announce"}, + {"udp://tracker.openbittorrent.com:6969/announce"}, +} + +var pieceLength = 100 * 1024 + +func NewArchiveManagerTorrent( + torrentConfig *params.TorrentConfig, + logger *zap.Logger, + persistence archivetypes.PersistenceProvider, + messaging *messaging.API, + identity *ecdsa.PrivateKey, + publisher archivetypes.HistoryArchivePublisher, +) *ArchiveManagerTorrent { + return &ArchiveManagerTorrent{ + torrentConfig: torrentConfig, + torrentTasks: make(map[string]metainfo.Hash), + + logger: logger, + persistence: persistence, + messaging: messaging, + identity: identity, + + publisher: publisher, + } +} + +// ArchiveServiceBackend interface implementation + +func (m *ArchiveManagerTorrent) SetOnline(online bool) { + if online { + err := m.Start() + if err != nil { + m.logger.Error("couldn't start torrent client", zap.Error(err)) + } + } +} + +func (m *ArchiveManagerTorrent) Start() error { + m.torrentMu.Lock() + defer m.torrentMu.Unlock() + + if m.torrentClientStartedLocked() { + return nil + } + + port, err := m.getTCPandUDPport(m.torrentConfig.Port) + if err != nil { + return err + } + + config := torrent.NewDefaultClientConfig() + config.SetListenAddr(":" + fmt.Sprint(port)) + config.Seed = true + + config.DataDir = m.torrentConfig.DataDir + + if _, err := os.Stat(m.torrentConfig.DataDir); os.IsNotExist(err) { + err := os.MkdirAll(m.torrentConfig.DataDir, 0700) + if err != nil { + return err + } + } + + m.logger.Info("Starting torrent client", zap.Any("port", port)) + // Instantiating the client will make it bootstrap and listen eagerly, + // so no go routine is needed here + client, err := torrent.NewClient(config) + if err != nil { + return err + } + m.torrentClient = client + return nil +} + +func (m *ArchiveManagerTorrent) Stop() error { + m.torrentMu.Lock() + defer m.torrentMu.Unlock() + + if m.torrentClientStartedLocked() { + m.logger.Info("Stopping torrent client") + errs := m.torrentClient.Close() + if len(errs) > 0 { + return errors.Join(errs...) + } + m.torrentClient = nil + } + return nil +} + +func (m *ArchiveManagerTorrent) IsStarted() bool { + m.torrentMu.RLock() + defer m.torrentMu.RUnlock() + return m.torrentClient != nil +} + +func (m *ArchiveManagerTorrent) SeedHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) error { + // NOTE: archiveLink is not currently used. We simply use the underlying + // torrent client to make sure we are seeding. + m.UnseedHistoryArchive(communityID, archiveLink) + + id := communityID.String() + torrentFile := torrentFile(m.torrentConfig.TorrentDir, id) + + metaInfo, err := metainfo.LoadFromFile(torrentFile) + if err != nil { + return err + } + + info, err := metaInfo.UnmarshalInfo() + if err != nil { + return err + } + + hash := metaInfo.HashInfoBytes() + m.setTorrentTask(id, hash) + + if err != nil { + return err + } + + client := m.getTorrentClient() + if client == nil { + return errors.New("torrent client is not started") + } + + torrent, err := client.AddTorrent(metaInfo) + if err != nil { + return err + } + + torrent.DownloadAll() + + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + HistoryArchivesSeedingSignal: &signal.HistoryArchivesSeedingSignal{ + CommunityID: communityID.String(), + }, + }) + + magnetLink := metaInfo.Magnet(nil, &info).String() + + m.logger.Debug("seeding torrent", zap.String("id", id), zap.String("magnetLink", magnetLink)) + return nil +} + +func (m *ArchiveManagerTorrent) UnseedHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) { + // NOTE: archiveLink is not currently used. We simply use torrentClient to drop + // the torrent corresponding to the communityID. + id := communityID.String() + + hash, exists := m.getTorrentTask(id) + + if exists { + client := m.getTorrentClient() + if client == nil { + return + } + torrent, ok := client.Torrent(hash) + if ok { + m.logger.Debug("Unseeding and dropping torrent for community: ", zap.Any("id", id)) + torrent.Drop() + m.deleteTorrentTask(id) + + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + HistoryArchivesUnseededSignal: &signal.HistoryArchivesUnseededSignal{ + CommunityID: id, + }, + }) + } + } +} + +func (m *ArchiveManagerTorrent) IsSeedingHistoryArchive(communityID cryptotypes.HexBytes, archiveLink string) bool { + // NOTE: archiveLink is not currently used. We simply use torrentClient to get + // the torrent corresponding to the communityID and check if it's seeding. + id := communityID.String() + hash, exists := m.getTorrentTask(id) + if !exists { + return false + } + + client := m.getTorrentClient() + if client == nil { + return false + } + torrent, ok := client.Torrent(hash) + return ok && torrent.Seeding() +} + +func (m *ArchiveManagerTorrent) DownloadHistoryArchives(communityID cryptotypes.HexBytes, archiveLink string, cancelTask chan struct{}) (*archivetypes.HistoryArchiveDownloadTaskInfo, error) { + id := communityID.String() + + ml, err := metainfo.ParseMagnetUri(archiveLink) + if err != nil { + return nil, err + } + + m.logger.Debug("adding torrent via magnetlink for community", zap.String("id", id), zap.String("magnetlink", archiveLink)) + client := m.getTorrentClient() + if client == nil { + return nil, errors.New("torrent client is not started") + } + + torrent, err := client.AddMagnet(archiveLink) + if err != nil { + return nil, err + } + + downloadTaskInfo := &archivetypes.HistoryArchiveDownloadTaskInfo{ + TotalDownloadedArchivesCount: 0, + TotalArchivesCount: 0, + Cancelled: false, + } + + m.setTorrentTask(id, ml.InfoHash) + timeout := time.After(20 * time.Second) + + m.logger.Debug("fetching torrent info", zap.String("magnetlink", archiveLink)) + select { + case <-timeout: + return nil, archivecommons.ErrArchiveTimedout + case <-cancelTask: + m.logger.Debug("cancelled fetching torrent info") + downloadTaskInfo.Cancelled = true + return downloadTaskInfo, nil + case <-torrent.GotInfo(): + + files := torrent.Files() + + i, ok := findIndexFile(files) + if !ok { + // We're dealing with a malformed torrent, so don't do anything + return nil, errors.New("malformed torrent data") + } + + indexFile := files[i] + indexFile.Download() + + m.logger.Debug("downloading history archive index") + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-cancelTask: + m.logger.Debug("cancelled downloading archive index") + downloadTaskInfo.Cancelled = true + return downloadTaskInfo, nil + case <-ticker.C: + if indexFile.BytesCompleted() == indexFile.Length() { + + index, err := m.loadHistoryArchiveIndexFromFile(m.identity, communityID) + if err != nil { + return nil, err + } + + existingArchiveIDs, err := m.persistence.GetDownloadedMessageArchiveIDs(communityID) + if err != nil { + return nil, err + } + + if len(existingArchiveIDs) == len(index.Archives) { + m.logger.Debug("download cancelled, no new archives") + return downloadTaskInfo, nil + } + + downloadTaskInfo.TotalDownloadedArchivesCount = len(existingArchiveIDs) + downloadTaskInfo.TotalArchivesCount = len(index.Archives) + + archiveHashes := make(archiveMDSlice, 0, downloadTaskInfo.TotalArchivesCount) + + for hash, metadata := range index.Archives { + archiveHashes = append(archiveHashes, &archiveMetadata{hash: hash, from: metadata.Metadata.From}) + } + + sort.Sort(sort.Reverse(archiveHashes)) + + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + DownloadingHistoryArchivesStartedSignal: &signal.DownloadingHistoryArchivesStartedSignal{ + CommunityID: communityID.String(), + }, + }) + + for _, hd := range archiveHashes { + + hash := hd.hash + hasArchive := false + + for _, existingHash := range existingArchiveIDs { + if existingHash == hash { + hasArchive = true + break + } + } + if hasArchive { + continue + } + + metadata := index.Archives[hash] + startIndex := int(metadata.Offset) / pieceLength + endIndex := startIndex + int(metadata.Size)/pieceLength + + downloadMsg := fmt.Sprintf("downloading data for message archive (%d/%d)", downloadTaskInfo.TotalDownloadedArchivesCount+1, downloadTaskInfo.TotalArchivesCount) + m.logger.Debug(downloadMsg, zap.String("hash", hash)) + m.logger.Debug("pieces (start, end)", zap.Any("startIndex", startIndex), zap.Any("endIndex", endIndex-1)) + torrent.DownloadPieces(startIndex, endIndex) + + piecesCompleted := make(map[int]bool) + for i = startIndex; i < endIndex; i++ { + piecesCompleted[i] = false + } + + psc := torrent.SubscribePieceStateChanges() + downloadTicker := time.NewTicker(1 * time.Second) + defer downloadTicker.Stop() + + downloadLoop: + for { + select { + case <-downloadTicker.C: + done := true + for i = startIndex; i < endIndex; i++ { + piecesCompleted[i] = torrent.PieceState(i).Complete + if !piecesCompleted[i] { + done = false + } + } + if done { + psc.Close() + break downloadLoop + } + case <-cancelTask: + m.logger.Debug("downloading archive data interrupted") + downloadTaskInfo.Cancelled = true + return downloadTaskInfo, nil + } + } + downloadTaskInfo.TotalDownloadedArchivesCount++ + err = m.persistence.SaveMessageArchiveID(communityID, hash) + if err != nil { + m.logger.Error("couldn't save message archive ID", zap.Error(err)) + continue + } + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + HistoryArchiveDownloadedSignal: &signal.HistoryArchiveDownloadedSignal{ + CommunityID: communityID.String(), + From: int(metadata.Metadata.From), + To: int(metadata.Metadata.To), + }, + }) + } + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + HistoryArchivesSeedingSignal: &signal.HistoryArchivesSeedingSignal{ + CommunityID: communityID.String(), + }, + }) + m.logger.Debug("finished downloading archives") + return downloadTaskInfo, nil + } + } + } + } +} + +func (m *ArchiveManagerTorrent) CreateHistoryArchiveFromMessages(communityID cryptotypes.HexBytes, messages []*messagingtypes.ReceivedMessage, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { + return m.createHistoryArchiveTorrent(communityID, messages, topics, startDate, endDate, partition, encrypt) +} + +func (m *ArchiveManagerTorrent) CreateHistoryArchiveFromDB(communityID cryptotypes.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { + return m.createHistoryArchiveTorrent(communityID, make([]*messagingtypes.ReceivedMessage, 0), topics, startDate, endDate, partition, encrypt) +} + +func (m *ArchiveManagerTorrent) CreateAndSeedHistoryArchive(communityID cryptotypes.HexBytes, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) error { + + archiveCreatedSuccessfully := true + m.UnseedHistoryArchive(communityID, "") + _, err := m.CreateHistoryArchiveFromDB(communityID, topics, startDate, endDate, partition, encrypt) + if err != nil { + archiveCreatedSuccessfully = false + m.logger.Error("failed to create history archive torrent", zap.Error(err)) + } else { + err = m.SeedHistoryArchive(communityID, "") + if err != nil { + archiveCreatedSuccessfully = false + m.logger.Error("failed to seed history archive torrent", zap.Error(err)) + } + } + + if !archiveCreatedSuccessfully { + return err + } + + return nil +} + +func (m *ArchiveManagerTorrent) LoadArchiveMessages(ctx context.Context, communityID cryptotypes.HexBytes, archiveLink string, downloadedArchiveID string) ([]*protobuf.WakuMessage, error) { + return m.extractMessagesFromHistoryArchive(communityID, downloadedArchiveID) +} + +// private methods +func (m *ArchiveManagerTorrent) archiveDataFile(communityID string) string { + return filepath.Join(m.torrentConfig.DataDir, communityID, "data") +} + +func (m *ArchiveManagerTorrent) extractMessagesFromHistoryArchive(communityID cryptotypes.HexBytes, downloadedArchiveID string) ([]*protobuf.WakuMessage, error) { + id := communityID.String() + + index, err := m.loadHistoryArchiveIndexFromFile(m.identity, communityID) + if err != nil { + return nil, err + } + + dataFile, err := os.Open(m.archiveDataFile(id)) + if err != nil { + return nil, err + } + defer dataFile.Close() + + m.logger.Debug("extracting messages from history archive", + zap.String("communityID", communityID.String()), + zap.String("downloadedArchiveID", downloadedArchiveID)) + metadata := index.Archives[downloadedArchiveID] + + _, err = dataFile.Seek(int64(metadata.Offset), 0) + if err != nil { + m.logger.Error("failed to seek archive data file", zap.Error(err)) + return nil, err + } + + data := make([]byte, metadata.Size-metadata.Padding) + m.logger.Debug("loading history archive data into memory", zap.Float64("data_size_MB", float64(metadata.Size-metadata.Padding)/1024.0/1024.0)) + _, err = dataFile.Read(data) + if err != nil { + m.logger.Error("failed failed to read archive data", zap.Error(err)) + return nil, err + } + + archive := &protobuf.WakuMessageArchive{} + + err = proto.Unmarshal(data, archive) + if err != nil { + pk, err := crypto.DecompressPubkey(communityID) + if err != nil { + m.logger.Error("failed to decompress community pubkey", zap.Error(err)) + return nil, err + } + + decryptedData, err := m.messaging.DecryptMessage(m.identity, pk, data) + if err != nil { + m.logger.Error("failed to decrypt message archive", zap.Error(err)) + return nil, err + } + + err = proto.Unmarshal(decryptedData, archive) + if err != nil { + m.logger.Error("failed to unmarshal message archive", zap.Error(err)) + return nil, err + } + } + return archive.Messages, nil +} + +func (m *ArchiveManagerTorrent) torrentClientStarted() bool { + m.torrentMu.RLock() + defer m.torrentMu.RUnlock() + return m.torrentClientStartedLocked() +} + +// Caller must hold m.torrentMu. +func (m *ArchiveManagerTorrent) torrentClientStartedLocked() bool { + return m.torrentClient != nil +} + +func (m *ArchiveManagerTorrent) getTorrentClient() *torrent.Client { + m.torrentMu.RLock() + defer m.torrentMu.RUnlock() + return m.torrentClient +} + +func (m *ArchiveManagerTorrent) getTorrentTask(communityID string) (metainfo.Hash, bool) { + m.torrentMu.RLock() + defer m.torrentMu.RUnlock() + hash, exists := m.torrentTasks[communityID] + return hash, exists +} + +func (m *ArchiveManagerTorrent) setTorrentTask(communityID string, hash metainfo.Hash) { + m.torrentMu.Lock() + defer m.torrentMu.Unlock() + m.torrentTasks[communityID] = hash +} + +func (m *ArchiveManagerTorrent) deleteTorrentTask(communityID string) { + m.torrentMu.Lock() + defer m.torrentMu.Unlock() + delete(m.torrentTasks, communityID) +} + +// getTCPandUDPport will return the same port number given if != 0, +// otherwise, it will attempt to find a free random tcp and udp port using +// the same number for both protocols +func (m *ArchiveManagerTorrent) getTCPandUDPport(portNumber int) (int, error) { + if portNumber != 0 { + return portNumber, nil + } + + // Find free port + for i := 0; i < 10; i++ { + port := func() int { + tcpAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort("localhost", "0")) + if err != nil { + m.logger.Warn("unable to resolve tcp addr: %v", zap.Error(err)) + return 0 + } + + tcpListener, err := net.ListenTCP("tcp", tcpAddr) + if err != nil { + m.logger.Warn("unable to listen on addr", zap.Stringer("addr", tcpAddr), zap.Error(err)) + return 0 + } + defer tcpListener.Close() + + port := tcpListener.Addr().(*net.TCPAddr).Port + + udpAddr, err := net.ResolveUDPAddr("udp", net.JoinHostPort("localhost", fmt.Sprintf("%d", port))) + if err != nil { + m.logger.Warn("unable to resolve udp addr: %v", zap.Error(err)) + return 0 + } + + udpListener, err := net.ListenUDP("udp", udpAddr) + if err != nil { + m.logger.Warn("unable to listen on addr", zap.Stringer("addr", udpAddr), zap.Error(err)) + return 0 + } + defer udpListener.Close() + + return port + }() + + if port != 0 { + return port, nil + } + } + + return 0, fmt.Errorf("no free port found") +} + +func (m *ArchiveManagerTorrent) loadHistoryArchiveIndexFromFile(myKey *ecdsa.PrivateKey, communityID cryptotypes.HexBytes) (*protobuf.WakuMessageArchiveIndex, error) { + wakuMessageArchiveIndexProto := &protobuf.WakuMessageArchiveIndex{} + + indexPath := m.archiveIndexFile(communityID.String()) + indexData, err := os.ReadFile(indexPath) + if err != nil { + return nil, err + } + + err = proto.Unmarshal(indexData, wakuMessageArchiveIndexProto) + if err != nil { + return nil, err + } + + if len(wakuMessageArchiveIndexProto.Archives) == 0 && len(indexData) > 0 { + // This means we're dealing with an encrypted index file, so we have to decrypt it first + pk, err := crypto.DecompressPubkey(communityID) + if err != nil { + return nil, err + } + + decryptedData, err := m.messaging.DecryptMessage(myKey, pk, indexData) + if err != nil { + m.logger.Error("failed to decrypt message archive", zap.Error(err)) + return nil, err + } + + err = proto.Unmarshal(decryptedData, wakuMessageArchiveIndexProto) + if err != nil { + return nil, err + } + } + + return wakuMessageArchiveIndexProto, nil +} + +func (m *ArchiveManagerTorrent) archiveIndexFile(communityID string) string { + return filepath.Join(m.torrentConfig.DataDir, communityID, "index") +} + +func (m *ArchiveManagerTorrent) createHistoryArchiveTorrent(communityID cryptotypes.HexBytes, msgs []*messagingtypes.ReceivedMessage, topics []messagingtypes.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { + + loadFromDB := len(msgs) == 0 + + from := startDate + to := from.Add(partition) + if to.After(endDate) { + to = endDate + } + + archiveDir := m.torrentConfig.DataDir + "/" + communityID.String() + torrentDir := m.torrentConfig.TorrentDir + indexPath := archiveDir + "/index" + dataPath := archiveDir + "/data" + + wakuMessageArchiveIndexProto := &protobuf.WakuMessageArchiveIndex{} + wakuMessageArchiveIndex := make(map[string]*protobuf.WakuMessageArchiveIndexMetadata) + archiveIDs := make([]string, 0) + + if _, err := os.Stat(archiveDir); os.IsNotExist(err) { + err := os.MkdirAll(archiveDir, 0700) + if err != nil { + return archiveIDs, err + } + } + if _, err := os.Stat(torrentDir); os.IsNotExist(err) { + err := os.MkdirAll(torrentDir, 0700) + if err != nil { + return archiveIDs, err + } + } + + _, err := os.Stat(indexPath) + if err == nil { + wakuMessageArchiveIndexProto, err = m.loadHistoryArchiveIndexFromFile(m.identity, communityID) + if err != nil { + return archiveIDs, err + } + } + + var offset uint64 = 0 + + for hash, metadata := range wakuMessageArchiveIndexProto.Archives { + offset = offset + metadata.Size + wakuMessageArchiveIndex[hash] = metadata + } + + var encodedArchives []*EncodedArchiveData + topicsAsByteArrays := archiveutils.TopicsAsByteArrays(topics) + + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{CreatingHistoryArchivesSignal: &signal.CreatingHistoryArchivesSignal{ + CommunityID: communityID.String(), + }}) + + m.logger.Debug("creating archives", + zap.Any("startDate", startDate), + zap.Any("endDate", endDate), + zap.Duration("partition", partition), + ) + for { + if from.Equal(endDate) || from.After(endDate) { + break + } + m.logger.Debug("creating message archive", + zap.Any("from", from), + zap.Any("to", to), + ) + + var messages []messagingtypes.ReceivedMessage + if loadFromDB { + messages, err = m.persistence.GetWakuMessagesByFilterTopic(topics, uint64(from.Unix()), uint64(to.Unix())) + if err != nil { + return archiveIDs, err + } + } else { + for _, msg := range msgs { + if int64(msg.Timestamp) >= from.Unix() && int64(msg.Timestamp) < to.Unix() { + messages = append(messages, *msg) + } + } + } + + if len(messages) == 0 { + // No need to create an archive with zero messages + m.logger.Debug("no messages in this partition") + from = to + to = to.Add(partition) + if to.After(endDate) { + to = endDate + } + continue + } + + m.logger.Debug("creating archive with messages", zap.Int("messagesCount", len(messages))) + + // Not only do we partition messages, we also chunk them + // roughly by size, such that each chunk will not exceed a given + // size and archive data doesn't get too big + messageChunks := make([][]messagingtypes.ReceivedMessage, 0) + currentChunkSize := 0 + currentChunk := make([]messagingtypes.ReceivedMessage, 0) + + for _, msg := range messages { + msgSize := len(msg.Payload) + len(msg.Sig) + if msgSize > archiveconsts.MaxArchiveSizeInBytes { + // we drop messages this big + continue + } + + if currentChunkSize+msgSize > archiveconsts.MaxArchiveSizeInBytes { + messageChunks = append(messageChunks, currentChunk) + currentChunk = make([]messagingtypes.ReceivedMessage, 0) + currentChunkSize = 0 + } + currentChunk = append(currentChunk, msg) + currentChunkSize = currentChunkSize + msgSize + } + messageChunks = append(messageChunks, currentChunk) + + for _, messages := range messageChunks { + wakuMessageArchive := m.createWakuMessageArchive(from, to, messages, topicsAsByteArrays) + encodedArchive, err := proto.Marshal(wakuMessageArchive) + if err != nil { + return archiveIDs, err + } + + if encrypt { + encodedArchive, err = m.messaging.BuildHashRatchetMessage(communityID, encodedArchive) + if err != nil { + return archiveIDs, err + } + } + + rawSize := len(encodedArchive) + padding := 0 + size := 0 + + if rawSize > pieceLength { + size = rawSize + pieceLength - (rawSize % pieceLength) + padding = size - rawSize + } else { + padding = pieceLength - rawSize + size = rawSize + padding + } + + wakuMessageArchiveIndexMetadata := &protobuf.WakuMessageArchiveIndexMetadata{ + Metadata: wakuMessageArchive.Metadata, + Offset: offset, + Size: uint64(size), + Padding: uint64(padding), + } + + wakuMessageArchiveIndexMetadataBytes, err := proto.Marshal(wakuMessageArchiveIndexMetadata) + if err != nil { + return archiveIDs, err + } + + archiveID := crypto.Keccak256Hash(wakuMessageArchiveIndexMetadataBytes).String() + archiveIDs = append(archiveIDs, archiveID) + wakuMessageArchiveIndex[archiveID] = wakuMessageArchiveIndexMetadata + encodedArchives = append(encodedArchives, &EncodedArchiveData{bytes: encodedArchive, padding: padding}) + offset = offset + uint64(rawSize) + uint64(padding) + } + + from = to + to = to.Add(partition) + if to.After(endDate) { + to = endDate + } + } + + if len(encodedArchives) > 0 { + + dataBytes := make([]byte, 0) + + for _, encodedArchiveData := range encodedArchives { + dataBytes = append(dataBytes, encodedArchiveData.bytes...) + dataBytes = append(dataBytes, make([]byte, encodedArchiveData.padding)...) + } + + wakuMessageArchiveIndexProto.Archives = wakuMessageArchiveIndex + indexBytes, err := proto.Marshal(wakuMessageArchiveIndexProto) + if err != nil { + return archiveIDs, err + } + + if encrypt { + indexBytes, err = m.messaging.BuildHashRatchetMessage(communityID, indexBytes) + if err != nil { + return archiveIDs, err + } + } + + err = os.WriteFile(indexPath, indexBytes, 0644) // nolint: gosec + if err != nil { + return archiveIDs, err + } + + file, err := os.OpenFile(dataPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + return archiveIDs, err + } + defer file.Close() + + _, err = file.Write(dataBytes) + if err != nil { + return archiveIDs, err + } + + metaInfo := metainfo.MetaInfo{ + AnnounceList: defaultAnnounceList, + } + metaInfo.SetDefaults() + metaInfo.CreatedBy = crypto.PubkeyToHex(&m.identity.PublicKey) + + info := metainfo.Info{ + PieceLength: int64(pieceLength), + } + + err = info.BuildFromFilePath(archiveDir) + if err != nil { + return archiveIDs, err + } + + metaInfo.InfoBytes, err = bencode.Marshal(info) + if err != nil { + return archiveIDs, err + } + + metaInfoBytes, err := bencode.Marshal(metaInfo) + if err != nil { + return archiveIDs, err + } + + err = os.WriteFile(torrentFile(m.torrentConfig.TorrentDir, communityID.String()), metaInfoBytes, 0644) // nolint: gosec + if err != nil { + return archiveIDs, err + } + + m.logger.Debug("torrent created", zap.Any("from", startDate.Unix()), zap.Any("to", endDate.Unix())) + + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + HistoryArchivesCreatedSignal: &signal.HistoryArchivesCreatedSignal{ + CommunityID: communityID.String(), + From: int(startDate.Unix()), + To: int(endDate.Unix()), + }, + }) + } else { + m.logger.Debug("no archives created") + m.publisher.Publish(&archivetypes.HistoryArchiveSignals{ + NoHistoryArchivesCreatedSignal: &signal.NoHistoryArchivesCreatedSignal{ + CommunityID: communityID.String(), + From: int(startDate.Unix()), + To: int(endDate.Unix()), + }, + }) + } + + lastMessageArchiveEndDate, err := m.persistence.GetLastMessageArchiveEndDate(communityID) + if err != nil { + return archiveIDs, err + } + + if lastMessageArchiveEndDate > 0 { + err = m.persistence.UpdateLastMessageArchiveEndDate(communityID, uint64(from.Unix())) + } else { + err = m.persistence.SaveLastMessageArchiveEndDate(communityID, uint64(from.Unix())) + } + if err != nil { + return archiveIDs, err + } + return archiveIDs, nil +} + +func (m *ArchiveManagerTorrent) createWakuMessageArchive(from time.Time, to time.Time, messages []messagingtypes.ReceivedMessage, topics [][]byte) *protobuf.WakuMessageArchive { + var wakuMessages []*protobuf.WakuMessage + + for _, msg := range messages { + wakuMessage := &protobuf.WakuMessage{ + Sig: msg.Sig, + Timestamp: uint64(msg.Timestamp), + Topic: msg.Topic.Bytes(), + Payload: msg.Payload, + Padding: msg.Padding, + Hash: msg.Hash, + ThirdPartyId: msg.ThirdPartyID, + } + wakuMessages = append(wakuMessages, wakuMessage) + } + + metadata := protobuf.WakuMessageArchiveMetadata{ + From: uint64(from.Unix()), + To: uint64(to.Unix()), + ContentTopic: topics, + } + + wakuMessageArchive := &protobuf.WakuMessageArchive{ + Metadata: &metadata, + Messages: wakuMessages, + } + return wakuMessageArchive +} + +// utility functions +func torrentFile(torrentDir, communityID string) string { + return filepath.Join(torrentDir, communityID+".torrent") +} + +func findIndexFile(files []*torrent.File) (index int, ok bool) { + for i, f := range files { + if f.DisplayPath() == "index" { + return i, true + } + } + return 0, false +} + +// Special functions +// These functions are not part of the ArchiveServiceBackend interface. +// Some legacy tests are accessing implementation details and for this reason +// we need to expose these special accessors. +func (m *ArchiveManagerTorrent) LoadHistoryArchiveIndexFromFile(myKey *ecdsa.PrivateKey, communityID cryptotypes.HexBytes) (*protobuf.WakuMessageArchiveIndex, error) { + return m.loadHistoryArchiveIndexFromFile(myKey, communityID) +} + +func (m *ArchiveManagerTorrent) GetTorrentFilePath(communityID string) string { + return torrentFile(m.torrentConfig.TorrentDir, communityID) +} + +func (m *ArchiveManagerTorrent) GetArchiveDataFilePath(communityID string) string { + return m.archiveDataFile(communityID) +} + +func (m *ArchiveManagerTorrent) GetArchiveIndexFilePath(communityID string) string { + return m.archiveIndexFile(communityID) +} + +func (m *ArchiveManagerTorrent) GetTorrentConfig() *params.TorrentConfig { + return m.torrentConfig +} + +func (m *ArchiveManagerTorrent) GetTorrentTasksCount() int { + m.torrentMu.RLock() + defer m.torrentMu.RUnlock() + return len(m.torrentTasks) +} + +func (m *ArchiveManagerTorrent) GetMetaInfoHashForCommunity(communityID string) metainfo.Hash { + hash, _ := m.getTorrentTask(communityID) + return hash +} + +func (m *ArchiveManagerTorrent) GetTorrentForCommunity(communityID string) (*torrent.Torrent, bool) { + hash, exists := m.getTorrentTask(communityID) + if !exists { + return nil, false + } + client := m.getTorrentClient() + if client == nil { + return nil, false + } + torrent, ok := client.Torrent(hash) + return torrent, ok +} diff --git a/protocol/communities/archive/torrent/archive_manager_torrent_test.go b/protocol/communities/archive/torrent/archive_manager_torrent_test.go new file mode 100644 index 00000000000..c6ac1ef4262 --- /dev/null +++ b/protocol/communities/archive/torrent/archive_manager_torrent_test.go @@ -0,0 +1,643 @@ +package torrent_test + +import ( + "os" + "testing" + "time" + + _ "github.com/mutecomm/go-sqlcipher/v4" // require go-sqlcipher that overrides default implementation + "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/proto" + + "github.com/status-im/status-go/internal/crypto" + "github.com/status-im/status-go/internal/db/appdatabase" + testutils2 "github.com/status-im/status-go/internal/testutils" + "github.com/status-im/status-go/params" + "github.com/status-im/status-go/pkg/messaging" + "github.com/status-im/status-go/pkg/messaging/types" + "github.com/status-im/status-go/protocol/communities" + "github.com/status-im/status-go/protocol/communities/archive" + archivetorrent "github.com/status-im/status-go/protocol/communities/archive/torrent" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" + "github.com/status-im/status-go/protocol/protobuf" + "github.com/status-im/status-go/protocol/requests" + "github.com/status-im/status-go/protocol/sqlite" +) + +func TestArchiveManagerTorrentSuite(t *testing.T) { + suite.Run(t, new(ArchiveManagerTorrentSuite)) +} + +type ArchiveManagerTorrentSuite struct { + suite.Suite + manager *communities.Manager + archiveService archive.ArchiveService +} + +type TimeSourceStub struct { +} + +func (t *TimeSourceStub) GetCurrentTime() uint64 { + return uint64(time.Now().Unix()) +} + +func buildTorrentConfig() *params.TorrentConfig { + return ¶ms.TorrentConfig{ + Enabled: true, + DataDir: os.TempDir() + "/archivedata", + TorrentDir: os.TempDir() + "/torrents", + Port: 0, + } +} + +func buildMessage(timestamp time.Time, topic types.ContentTopic, hash []byte) types.ReceivedMessage { + message := types.ReceivedMessage{ + Sig: []byte{1}, + Timestamp: uint32(timestamp.Unix()), + Topic: topic, + Payload: []byte{1}, + Padding: []byte{1}, + Hash: hash, + } + return message +} + +func (s *ArchiveManagerTorrentSuite) buildCommunityWithChat() (*communities.Community, string, error) { + createRequest := &requests.CreateCommunity{ + Name: "status", + Description: "status community description", + Membership: protobuf.CommunityPermissions_AUTO_ACCEPT, + } + community, err := s.manager.CreateCommunity(createRequest, true) + if err != nil { + return nil, "", err + } + chat := &protobuf.CommunityChat{ + Identity: &protobuf.ChatIdentity{ + DisplayName: "added-chat", + Description: "description", + }, + Permissions: &protobuf.CommunityPermissions{ + Access: protobuf.CommunityPermissions_AUTO_ACCEPT, + }, + Members: make(map[string]*protobuf.CommunityMember), + } + changes, err := s.manager.CreateChat(community.ID(), chat, true, "") + if err != nil { + return nil, "", err + } + + chatID := "" + for cID := range changes.ChatsAdded { + chatID = cID + break + } + return community, chatID, nil +} + +func (s *ArchiveManagerTorrentSuite) buildManagers(ownerVerifier communities.OwnerVerifier) (*communities.Manager, archive.ArchiveService) { + db, err := testutils2.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) + s.Require().NoError(err, "creating sqlite db instance") + err = sqlite.Migrate(db) + s.Require().NoError(err, "protocol migrate") + + key, err := crypto.GenerateKey() + s.Require().NoError(err) + + logger := testutils2.MustCreateTestLogger() + + m, err := communities.NewManager(key, "", db, logger, nil, ownerVerifier, nil, &TimeSourceStub{}, nil, nil) + s.Require().NoError(err) + s.Require().NoError(m.Start()) + + amc := &archivetypes.ArchiveManagerConfig{ + TorrentConfig: buildTorrentConfig(), + Logger: logger, + Persistence: m.GetPersistence(), + Messaging: nil, + Identity: key, + Publisher: m, + } + t := archive.NewArchiveManager(amc) + s.Require().NoError(err) + + return m, t +} + +func (s *ArchiveManagerTorrentSuite) getArchiveManager() *archive.ArchiveManager { + archiveManager, ok := s.archiveService.(*archive.ArchiveManager) + s.Require().True(ok) + return archiveManager +} + +func (s *ArchiveManagerTorrentSuite) getTorrentBackend() *archivetorrent.ArchiveManagerTorrent { + archiveManager := s.getArchiveManager() + backend, err := archiveManager.GetTorrentBackend() + s.Require().NoError(err) + return backend +} + +func (s *ArchiveManagerTorrentSuite) getTorrentConfig() *params.TorrentConfig { + return s.getTorrentBackend().GetTorrentConfig() +} + +func (s *ArchiveManagerTorrentSuite) getTorrentFilePath(communityID string) string { + return s.getTorrentBackend().GetTorrentFilePath(communityID) +} + +func (s *ArchiveManagerTorrentSuite) SetupTest() { + m, t := s.buildManagers(nil) + communities.SetValidateInterval(30 * time.Millisecond) + s.manager = m + s.archiveService = t +} + +func (s *ArchiveManagerTorrentSuite) TestStartAndStopTorrentClient() { + err := s.archiveService.Start() + s.Require().NoError(err) + s.Require().True(s.archiveService.IsStarted()) + defer s.archiveService.Stop() //nolint: errcheck + + torrentConfig := s.getTorrentConfig() + + _, err = os.Stat(torrentConfig.DataDir) + s.Require().NoError(err) +} + +func (s *ArchiveManagerTorrentSuite) TestStartHistoryArchiveTasksInterval() { + err := s.archiveService.Start() + s.Require().NoError(err) + defer s.archiveService.Stop() //nolint: errcheck + + community, _, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + interval := 10 * time.Second + go s.archiveService.StartHistoryArchiveTasksInterval(community.ID(), community.UniversalChatID(), community.Encrypted(), interval) + // Due to async exec we need to wait a bit until we check + // the task count. + time.Sleep(5 * time.Second) + + count := s.archiveService.GetHistoryTasksCount() + s.Require().Equal(count, 1) + + // We wait another 5 seconds to ensure the first tick has kicked in + time.Sleep(5 * time.Second) + + _, err = os.Stat(s.getTorrentFilePath(community.IDString())) + s.Require().Error(err) + + s.archiveService.StopHistoryArchiveTasksInterval(community.ID()) + + archiveManager := s.getArchiveManager() + archiveManager.Wait() + count = s.archiveService.GetHistoryTasksCount() + s.Require().Equal(count, 0) +} + +func (s *ArchiveManagerTorrentSuite) TestStopHistoryArchiveTasksIntervals() { + err := s.archiveService.Start() + s.Require().NoError(err) + defer s.archiveService.Stop() //nolint: errcheck + + community, _, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + interval := 10 * time.Second + go s.archiveService.StartHistoryArchiveTasksInterval(community.ID(), community.UniversalChatID(), community.Encrypted(), interval) + + time.Sleep(2 * time.Second) + + count := s.archiveService.GetHistoryTasksCount() + s.Require().Equal(count, 1) + + archiveManager := s.getArchiveManager() + archiveManager.StopHistoryArchiveTasksIntervalsAndWait() + + count = s.archiveService.GetHistoryTasksCount() + s.Require().Equal(count, 0) +} + +func (s *ArchiveManagerTorrentSuite) TestStopTorrentClient_ShouldStopHistoryArchiveTasks() { + err := s.archiveService.Start() + s.Require().NoError(err) + defer s.archiveService.Stop() //nolint: errcheck + + community, _, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + interval := 10 * time.Second + go s.archiveService.StartHistoryArchiveTasksInterval(community.ID(), community.UniversalChatID(), community.Encrypted(), interval) + // Due to async exec we need to wait a bit until we check + // the task count. + time.Sleep(2 * time.Second) + + count := s.archiveService.GetHistoryTasksCount() + s.Require().Equal(count, 1) + + err = s.archiveService.Stop() + s.Require().NoError(err) + + count = s.archiveService.GetHistoryTasksCount() + s.Require().Equal(count, 0) +} + +func (s *ArchiveManagerTorrentSuite) TestStartTorrentClient_DelayedUntilOnline() { + s.Require().False(s.archiveService.IsStarted()) + + s.archiveService.SetOnline(true) + s.Require().True(s.archiveService.IsStarted()) +} + +func (s *ArchiveManagerTorrentSuite) TestCreateHistoryArchiveTorrent_WithoutMessages() { + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + // Time range of 7 days + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + // Partition of 7 days + partition := 7 * 24 * time.Hour + + torrentBackend := s.getTorrentBackend() + + _, err = torrentBackend.CreateHistoryArchiveFromDB(community.ID(), topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + // There are no waku messages in the database so we don't expect + // any archives to be created + _, err = os.Stat(torrentBackend.GetArchiveDataFilePath(community.IDString())) + s.Require().Error(err) + _, err = os.Stat(torrentBackend.GetArchiveIndexFilePath(community.IDString())) + s.Require().Error(err) + _, err = os.Stat(torrentBackend.GetTorrentFilePath(community.IDString())) + s.Require().Error(err) +} + +func (s *ArchiveManagerTorrentSuite) TestCreateHistoryArchiveTorrent_ShouldCreateArchive() { + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + // Time range of 7 days + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + // Partition of 7 days, this should create a single archive + partition := 7 * 24 * time.Hour + + message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) + message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) + // This message is outside of the startDate-endDate range and should not + // be part of the archive + message3 := buildMessage(endDate.Add(2*time.Hour), topic, []byte{3}) + + err = s.manager.StoreWakuMessage(&message1) + s.Require().NoError(err) + err = s.manager.StoreWakuMessage(&message2) + s.Require().NoError(err) + err = s.manager.StoreWakuMessage(&message3) + s.Require().NoError(err) + + torrentBackend := s.getTorrentBackend() + + _, err = torrentBackend.CreateHistoryArchiveFromDB(community.ID(), topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + _, err = os.Stat(torrentBackend.GetArchiveDataFilePath(community.IDString())) + s.Require().NoError(err) + _, err = os.Stat(torrentBackend.GetArchiveIndexFilePath(community.IDString())) + s.Require().NoError(err) + _, err = os.Stat(torrentBackend.GetTorrentFilePath(community.IDString())) + s.Require().NoError(err) + + index, err := torrentBackend.LoadHistoryArchiveIndexFromFile(s.manager.GetIdentity(), community.ID()) + s.Require().NoError(err) + s.Require().Len(index.Archives, 1) + + totalData, err := os.ReadFile(torrentBackend.GetArchiveDataFilePath(community.IDString())) + s.Require().NoError(err) + + for _, metadata := range index.Archives { + archive := &protobuf.WakuMessageArchive{} + data := totalData[metadata.Offset : metadata.Offset+metadata.Size-metadata.Padding] + + err = proto.Unmarshal(data, archive) + s.Require().NoError(err) + + s.Require().Len(archive.Messages, 2) + } +} + +func (s *ArchiveManagerTorrentSuite) TestCreateHistoryArchiveTorrent_ShouldCreateMultipleArchives() { + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + // Time range of 3 weeks + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 21, 00, 00, 00, 0, time.UTC) + // 7 days partition, this should create three archives + partition := 7 * 24 * time.Hour + + message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) + message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) + // We expect 2 archives to be created for startDate - endDate of each + // 7 days of data. This message should end up in the second archive + message3 := buildMessage(startDate.Add(8*24*time.Hour), topic, []byte{3}) + // This one should end up in the third archive + message4 := buildMessage(startDate.Add(14*24*time.Hour), topic, []byte{4}) + + err = s.manager.StoreWakuMessage(&message1) + s.Require().NoError(err) + err = s.manager.StoreWakuMessage(&message2) + s.Require().NoError(err) + err = s.manager.StoreWakuMessage(&message3) + s.Require().NoError(err) + err = s.manager.StoreWakuMessage(&message4) + s.Require().NoError(err) + + torrentBackend := s.getTorrentBackend() + + _, err = torrentBackend.CreateHistoryArchiveFromDB(community.ID(), topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + index, err := torrentBackend.LoadHistoryArchiveIndexFromFile(s.manager.GetIdentity(), community.ID()) + s.Require().NoError(err) + s.Require().Len(index.Archives, 3) + + totalData, err := os.ReadFile(torrentBackend.GetArchiveDataFilePath(community.IDString())) + s.Require().NoError(err) + + // First archive has 2 messages + // Second archive has 1 message + // Third archive has 1 message + fromMap := map[uint64]int{ + uint64(startDate.Unix()): 2, + uint64(startDate.Add(partition).Unix()): 1, + uint64(startDate.Add(partition * 2).Unix()): 1, + } + + for _, metadata := range index.Archives { + archive := &protobuf.WakuMessageArchive{} + data := totalData[metadata.Offset : metadata.Offset+metadata.Size-metadata.Padding] + + err = proto.Unmarshal(data, archive) + s.Require().NoError(err) + s.Require().Len(archive.Messages, fromMap[metadata.Metadata.From]) + } +} + +func (s *ArchiveManagerTorrentSuite) TestCreateHistoryArchiveTorrent_ShouldAppendArchives() { + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + // Time range of 1 week + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + // 7 days partition, this should create one archive + partition := 7 * 24 * time.Hour + + message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) + err = s.manager.StoreWakuMessage(&message1) + s.Require().NoError(err) + + torrentBackend := s.getTorrentBackend() + + _, err = torrentBackend.CreateHistoryArchiveFromDB(community.ID(), topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + index, err := torrentBackend.LoadHistoryArchiveIndexFromFile(s.manager.GetIdentity(), community.ID()) + s.Require().NoError(err) + s.Require().Len(index.Archives, 1) + + // Time range of next week + startDate = time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + endDate = time.Date(2020, 1, 14, 00, 00, 00, 0, time.UTC) + + message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) + err = s.manager.StoreWakuMessage(&message2) + s.Require().NoError(err) + + _, err = torrentBackend.CreateHistoryArchiveFromDB(community.ID(), topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + index, err = torrentBackend.LoadHistoryArchiveIndexFromFile(s.manager.GetIdentity(), community.ID()) + s.Require().NoError(err) + s.Require().Len(index.Archives, 2) +} + +func (s *ArchiveManagerTorrentSuite) TestCreateHistoryArchiveTorrentFromMessages() { + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + // Time range of 7 days + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + // Partition of 7 days, this should create a single archive + partition := 7 * 24 * time.Hour + + message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) + message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) + // This message is outside of the startDate-endDate range and should not + // be part of the archive + message3 := buildMessage(endDate.Add(2*time.Hour), topic, []byte{3}) + + torrentBackend := s.getTorrentBackend() + + _, err = torrentBackend.CreateHistoryArchiveFromMessages(community.ID(), []*types.ReceivedMessage{&message1, &message2, &message3}, topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + _, err = os.Stat(torrentBackend.GetArchiveDataFilePath(community.IDString())) + s.Require().NoError(err) + _, err = os.Stat(torrentBackend.GetArchiveIndexFilePath(community.IDString())) + s.Require().NoError(err) + _, err = os.Stat(torrentBackend.GetTorrentFilePath(community.IDString())) + s.Require().NoError(err) + + index, err := torrentBackend.LoadHistoryArchiveIndexFromFile(s.manager.GetIdentity(), community.ID()) + s.Require().NoError(err) + s.Require().Len(index.Archives, 1) + + totalData, err := os.ReadFile(torrentBackend.GetArchiveDataFilePath(community.IDString())) + s.Require().NoError(err) + + for _, metadata := range index.Archives { + archive := &protobuf.WakuMessageArchive{} + data := totalData[metadata.Offset : metadata.Offset+metadata.Size-metadata.Padding] + + err = proto.Unmarshal(data, archive) + s.Require().NoError(err) + + s.Require().Len(archive.Messages, 2) + } +} + +func (s *ArchiveManagerTorrentSuite) TestCreateHistoryArchiveTorrentFromMessages_ShouldCreateMultipleArchives() { + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + // Time range of 3 weeks + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 21, 00, 00, 00, 0, time.UTC) + // 7 days partition, this should create three archives + partition := 7 * 24 * time.Hour + + message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) + message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) + // We expect 2 archives to be created for startDate - endDate of each + // 7 days of data. This message should end up in the second archive + message3 := buildMessage(startDate.Add(8*24*time.Hour), topic, []byte{3}) + // This one should end up in the third archive + message4 := buildMessage(startDate.Add(14*24*time.Hour), topic, []byte{4}) + + torrentBackend := s.getTorrentBackend() + _, err = torrentBackend.CreateHistoryArchiveFromMessages(community.ID(), []*types.ReceivedMessage{&message1, &message2, &message3, &message4}, topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + index, err := torrentBackend.LoadHistoryArchiveIndexFromFile(s.manager.GetIdentity(), community.ID()) + s.Require().NoError(err) + s.Require().Len(index.Archives, 3) + + totalData, err := os.ReadFile(torrentBackend.GetArchiveDataFilePath(community.IDString())) + s.Require().NoError(err) + + // First archive has 2 messages + // Second archive has 1 message + // Third archive has 1 message + fromMap := map[uint64]int{ + uint64(startDate.Unix()): 2, + uint64(startDate.Add(partition).Unix()): 1, + uint64(startDate.Add(partition * 2).Unix()): 1, + } + + for _, metadata := range index.Archives { + archive := &protobuf.WakuMessageArchive{} + data := totalData[metadata.Offset : metadata.Offset+metadata.Size-metadata.Padding] + + err = proto.Unmarshal(data, archive) + s.Require().NoError(err) + s.Require().Len(archive.Messages, fromMap[metadata.Metadata.From]) + } +} + +func (s *ArchiveManagerTorrentSuite) TestCreateHistoryArchiveTorrentFromMessages_ShouldAppendArchives() { + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + // Time range of 1 week + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + // 7 days partition, this should create one archive + partition := 7 * 24 * time.Hour + + message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) + + torrentBackend := s.getTorrentBackend() + + _, err = torrentBackend.CreateHistoryArchiveFromMessages(community.ID(), []*types.ReceivedMessage{&message1}, topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + index, err := torrentBackend.LoadHistoryArchiveIndexFromFile(s.manager.GetIdentity(), community.ID()) + s.Require().NoError(err) + s.Require().Len(index.Archives, 1) + + // Time range of next week + startDate = time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + endDate = time.Date(2020, 1, 14, 00, 00, 00, 0, time.UTC) + + message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) + + _, err = torrentBackend.CreateHistoryArchiveFromMessages(community.ID(), []*types.ReceivedMessage{&message2}, topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + index, err = torrentBackend.LoadHistoryArchiveIndexFromFile(s.manager.GetIdentity(), community.ID()) + s.Require().NoError(err) + s.Require().Len(index.Archives, 2) +} + +func (s *ArchiveManagerTorrentSuite) TestSeedHistoryArchiveTorrent() { + err := s.archiveService.Start() + s.Require().NoError(err) + defer s.archiveService.Stop() //nolint: errcheck + + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + partition := 7 * 24 * time.Hour + + message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) + err = s.manager.StoreWakuMessage(&message1) + s.Require().NoError(err) + + torrentBackend := s.getTorrentBackend() + + _, err = torrentBackend.CreateHistoryArchiveFromDB(community.ID(), topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + err = s.archiveService.SeedHistoryArchive(community.ID(), "") + s.Require().NoError(err) + s.Require().Equal(torrentBackend.GetTorrentTasksCount(), 1) + + torrent, ok := torrentBackend.GetTorrentForCommunity(community.IDString()) + defer torrent.Drop() + + s.Require().Equal(ok, true) + s.Require().Equal(torrent.Seeding(), true) +} + +func (s *ArchiveManagerTorrentSuite) TestUnseedHistoryArchiveTorrent() { + err := s.archiveService.Start() + s.Require().NoError(err) + defer s.archiveService.Stop() //nolint: errcheck + + community, chatID, err := s.buildCommunityWithChat() + s.Require().NoError(err) + + topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) + topics := []types.ContentTopic{topic} + + startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) + endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) + partition := 7 * 24 * time.Hour + + message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) + err = s.manager.StoreWakuMessage(&message1) + s.Require().NoError(err) + + torrentBackend := s.getTorrentBackend() + + _, err = torrentBackend.CreateHistoryArchiveFromDB(community.ID(), topics, startDate, endDate, partition, false) + s.Require().NoError(err) + + err = torrentBackend.SeedHistoryArchive(community.ID(), "") + s.Require().NoError(err) + s.Require().Equal(torrentBackend.GetTorrentTasksCount(), 1) + + s.archiveService.UnseedHistoryArchive(community.ID(), "") + _, ok := torrentBackend.GetTorrentForCommunity(community.IDString()) + s.Require().Equal(ok, false) +} diff --git a/protocol/communities/archive/types/archive_types.go b/protocol/communities/archive/types/archive_types.go new file mode 100644 index 00000000000..00de30d1e84 --- /dev/null +++ b/protocol/communities/archive/types/archive_types.go @@ -0,0 +1,81 @@ +package types + +import ( + "crypto/ecdsa" + "sync" + + "go.uber.org/zap" + + cryptotypes "github.com/status-im/status-go/internal/crypto/types" + "github.com/status-im/status-go/params" + "github.com/status-im/status-go/pkg/messaging" + messagingtypes "github.com/status-im/status-go/pkg/messaging/types" + "github.com/status-im/status-go/signal" +) + +type HistoryArchiveSignals struct { + CreatingHistoryArchivesSignal *signal.CreatingHistoryArchivesSignal + HistoryArchivesCreatedSignal *signal.HistoryArchivesCreatedSignal + NoHistoryArchivesCreatedSignal *signal.NoHistoryArchivesCreatedSignal + HistoryArchivesSeedingSignal *signal.HistoryArchivesSeedingSignal + HistoryArchivesUnseededSignal *signal.HistoryArchivesUnseededSignal + HistoryArchiveDownloadedSignal *signal.HistoryArchiveDownloadedSignal + DownloadingHistoryArchivesStartedSignal *signal.DownloadingHistoryArchivesStartedSignal + DownloadingHistoryArchivesFinishedSignal *signal.DownloadingHistoryArchivesFinishedSignal + ImportingHistoryArchiveMessagesSignal *signal.ImportingHistoryArchiveMessagesSignal +} + +type HistoryArchiveDownloadTask struct { + CancelChan chan struct{} + Waiter sync.WaitGroup + m sync.RWMutex + Cancelled bool +} + +func (t *HistoryArchiveDownloadTask) IsCancelled() bool { + t.m.RLock() + defer t.m.RUnlock() + return t.Cancelled +} + +func (t *HistoryArchiveDownloadTask) Cancel() { + t.m.Lock() + defer t.m.Unlock() + t.Cancelled = true + close(t.CancelChan) +} + +type HistoryArchiveDownloadTaskInfo struct { + TotalDownloadedArchivesCount int + TotalArchivesCount int + Cancelled bool +} + +type PersistenceProvider interface { + GetDownloadedMessageArchiveIDs(communityID cryptotypes.HexBytes) ([]string, error) + SaveMessageArchiveID(communityID cryptotypes.HexBytes, hash string) error + GetWakuMessagesByFilterTopic(topics []messagingtypes.ContentTopic, from uint64, to uint64) ([]messagingtypes.ReceivedMessage, error) + GetLastMessageArchiveEndDate(communityID cryptotypes.HexBytes) (uint64, error) + UpdateLastMessageArchiveEndDate(communityID cryptotypes.HexBytes, endDate uint64) error + SaveLastMessageArchiveEndDate(communityID cryptotypes.HexBytes, endDate uint64) error + GetLastSeenArchiveLink(communityID cryptotypes.HexBytes) (string, error) + UpdateLastSeenArchiveLink(communityID cryptotypes.HexBytes, archiveLink string) error + GetCommunityChatIDs(communityID cryptotypes.HexBytes) ([]string, error) + CommunityExists(memberIdentity *ecdsa.PublicKey, id []byte) (bool, error) + GetOldestWakuMessageTimestamp(topics []messagingtypes.ContentTopic) (uint64, error) + GetMessageArchiveIDsToImport(communityID cryptotypes.HexBytes) ([]string, error) + SetMessageArchiveIDImported(communityID cryptotypes.HexBytes, hash string, imported bool) error +} + +type HistoryArchivePublisher interface { + Publish(subscription *HistoryArchiveSignals) +} + +type ArchiveManagerConfig struct { + TorrentConfig *params.TorrentConfig + Logger *zap.Logger + Persistence PersistenceProvider + Messaging *messaging.API + Identity *ecdsa.PrivateKey + Publisher HistoryArchivePublisher +} diff --git a/protocol/communities/archive/utils/archive_utils.go b/protocol/communities/archive/utils/archive_utils.go new file mode 100644 index 00000000000..bbf421de2d8 --- /dev/null +++ b/protocol/communities/archive/utils/archive_utils.go @@ -0,0 +1,22 @@ +package utils + +import ( + "github.com/status-im/status-go/internal/crypto/types" + messagingtypes "github.com/status-im/status-go/pkg/messaging/types" +) + +func TopicsAsByteArrays(topics []messagingtypes.ContentTopic) [][]byte { + var topicsAsByteArrays [][]byte + for _, t := range topics { + topicsAsByteArrays = append(topicsAsByteArrays, t.Bytes()) + } + return topicsAsByteArrays +} + +// UniversalChatIDFromCommunityID constructs the universal chat ID from a community ID. +// This is used as a content-topic for all chats in the community. +// It corresponds to Community.UniversalChatID() which returns Community.MemberUpdateChannelID() +// which is IDString() + "-memberUpdate". +func UniversalChatIDFromCommunityID(communityID types.HexBytes) string { + return types.EncodeHex(communityID) + "-memberUpdate" +} diff --git a/protocol/communities/manager.go b/protocol/communities/manager.go index 3a564593b12..c6e34938dd1 100644 --- a/protocol/communities/manager.go +++ b/protocol/communities/manager.go @@ -34,7 +34,6 @@ import ( types3 "github.com/status-im/status-go/internal/crypto/types" multiaccountscommon "github.com/status-im/status-go/internal/db/multiaccounts/common" "github.com/status-im/status-go/internal/images" - "github.com/status-im/status-go/params" "github.com/status-im/status-go/pkg/messaging" types2 "github.com/status-im/status-go/pkg/messaging/types" "github.com/status-im/status-go/protocol/common" @@ -49,20 +48,14 @@ import ( "github.com/status-im/status-go/services/wallet/thirdparty" tokentypes "github.com/status-im/status-go/services/wallet/token/types" "github.com/status-im/status-go/signal" + + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" ) type Publisher interface { publish(subscription *Subscription) } -var defaultAnnounceList = [][]string{ - {"udp://tracker.opentrackr.org:1337/announce"}, - {"udp://tracker.openbittorrent.com:6969/announce"}, -} -var pieceLength = 100 * 1024 - -const maxArchiveSizeInBytes = 30000000 - var maxNbMembers = 5000 var maxNbPendingRequestedMembers = 100 @@ -169,76 +162,6 @@ func (c *CommunityLock) Init() { c.locks = make(map[string]*sync.Mutex) } -type HistoryArchiveDownloadTask struct { - CancelChan chan struct{} - Waiter sync.WaitGroup - m sync.RWMutex - Cancelled bool -} - -type HistoryArchiveDownloadTaskInfo struct { - TotalDownloadedArchivesCount int - TotalArchivesCount int - Cancelled bool -} - -type ArchiveFileService interface { - CreateHistoryArchiveTorrentFromMessages(communityID types3.HexBytes, messages []*types2.ReceivedMessage, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) - CreateHistoryArchiveTorrentFromDB(communityID types3.HexBytes, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) - SaveMessageArchiveID(communityID types3.HexBytes, hash string) error - GetMessageArchiveIDsToImport(communityID types3.HexBytes) ([]string, error) - SetMessageArchiveIDImported(communityID types3.HexBytes, hash string, imported bool) error - ExtractMessagesFromHistoryArchive(communityID types3.HexBytes, archiveID string) ([]*protobuf.WakuMessage, error) - GetHistoryArchiveMagnetlink(communityID types3.HexBytes) (string, error) - LoadHistoryArchiveIndexFromFile(myKey *ecdsa.PrivateKey, communityID types3.HexBytes) (*protobuf.WakuMessageArchiveIndex, error) -} - -type ArchiveService interface { - ArchiveFileService - - SetPaused(paused bool) - SetOnline(bool) - SetTorrentConfig(*params.TorrentConfig) - StartTorrentClient() error - Stop() error - IsReady() bool - GetCommunityChatsFilters(communityID types3.HexBytes) (types2.ChatFilters, error) - GetCommunityChatsTopics(communityID types3.HexBytes) ([]types2.ContentTopic, error) - GetHistoryArchivePartitionStartTimestamp(communityID types3.HexBytes) (uint64, error) - CreateAndSeedHistoryArchive(communityID types3.HexBytes, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) error - StartHistoryArchiveTasksInterval(community *Community, interval time.Duration) - StopHistoryArchiveTasksInterval(communityID types3.HexBytes) - SeedHistoryArchiveTorrent(communityID types3.HexBytes) error - UnseedHistoryArchiveTorrent(communityID types3.HexBytes) - IsSeedingHistoryArchiveTorrent(communityID types3.HexBytes) bool - GetHistoryArchiveDownloadTask(communityID string) *HistoryArchiveDownloadTask - AddHistoryArchiveDownloadTask(communityID string, task *HistoryArchiveDownloadTask) - DownloadHistoryArchivesByMagnetlink(communityID types3.HexBytes, magnetlink string, cancelTask chan struct{}) (*HistoryArchiveDownloadTaskInfo, error) - TorrentFileExists(communityID string) bool -} - -type ArchiveManagerConfig struct { - TorrentConfig *params.TorrentConfig - Logger *zap.Logger - Persistence *Persistence - Messaging *messaging.API - Identity *ecdsa.PrivateKey - Publisher Publisher -} - -func (t *HistoryArchiveDownloadTask) IsCancelled() bool { - t.m.RLock() - defer t.m.RUnlock() - return t.Cancelled -} - -func (t *HistoryArchiveDownloadTask) Cancel() { - t.m.Lock() - defer t.m.Unlock() - t.Cancelled = true - close(t.CancelChan) -} - type membersReevaluationTask struct { lastStartTime time.Time lastSuccessTime time.Time @@ -468,6 +391,10 @@ func NewManager( return manager, nil } +func (m *Manager) GetIdentity() *ecdsa.PrivateKey { + return m.identity +} + func (m *Manager) SetMediaServerProperties() { m.mediaServer.SetCommunityImageVersionReader(func(communityID string) uint32 { return m.communityImageVersions[communityID] @@ -489,21 +416,13 @@ func (m *Manager) SetMediaServerProperties() { } type Subscription struct { - Community *Community - CreatingHistoryArchivesSignal *signal.CreatingHistoryArchivesSignal - HistoryArchivesCreatedSignal *signal.HistoryArchivesCreatedSignal - NoHistoryArchivesCreatedSignal *signal.NoHistoryArchivesCreatedSignal - HistoryArchivesSeedingSignal *signal.HistoryArchivesSeedingSignal - HistoryArchivesUnseededSignal *signal.HistoryArchivesUnseededSignal - HistoryArchiveDownloadedSignal *signal.HistoryArchiveDownloadedSignal - DownloadingHistoryArchivesStartedSignal *signal.DownloadingHistoryArchivesStartedSignal - DownloadingHistoryArchivesFinishedSignal *signal.DownloadingHistoryArchivesFinishedSignal - ImportingHistoryArchiveMessagesSignal *signal.ImportingHistoryArchiveMessagesSignal - CommunityEventsMessage *CommunityEventsMessage - AcceptedRequestsToJoin []types3.HexBytes - RejectedRequestsToJoin []types3.HexBytes - CommunityPrivilegedMemberSyncMessage *CommunityPrivilegedMemberSyncMessage - TokenCommunityValidated *CommunityResponse + archivetypes.HistoryArchiveSignals + Community *Community + CommunityEventsMessage *CommunityEventsMessage + AcceptedRequestsToJoin []types3.HexBytes + RejectedRequestsToJoin []types3.HexBytes + CommunityPrivilegedMemberSyncMessage *CommunityPrivilegedMemberSyncMessage + TokenCommunityValidated *CommunityResponse } type CommunityResponse struct { @@ -750,6 +669,10 @@ func (m *Manager) publish(subscription *Subscription) { } } +func (m *Manager) Publish(subscription *archivetypes.HistoryArchiveSignals) { + m.publish(&Subscription{HistoryArchiveSignals: *subscription}) +} + func (m *Manager) All() ([]*Community, error) { return m.persistence.AllCommunities(&m.identity.PublicKey) } @@ -2258,19 +2181,26 @@ func (m *Manager) handleCommunityDescriptionMessageCommon(community *Community, return nil, err } - cdMagnetlinkClock := community.config.CommunityDescription.ArchiveMagnetlinkClock + cdArchiveLinkClock := community.config.CommunityDescription.ArchiveLinkClock + + m.logger.Debug("[LogosStorage][handleCommunityDescription] handling community description archive info", + zap.String("communityID", community.IDString()), + zap.Uint64("archiveLinkClock", cdArchiveLinkClock), + ) + if !hasCommunityArchiveInfo { - err = m.persistence.SaveCommunityArchiveInfo(community.ID(), cdMagnetlinkClock, 0) + m.logger.Debug("[LogosStorage][handleCommunityDescription] saving community archive info: hasCommunityArchiveInfo=false") + err = m.persistence.SaveCommunityArchiveInfo(community.ID(), cdArchiveLinkClock, 0) if err != nil { return nil, err } } else { - magnetlinkClock, err := m.persistence.GetMagnetlinkMessageClock(community.ID()) + archiveLinkClock, err := m.persistence.GetArchiveLinkMessageClock(community.ID()) if err != nil { return nil, err } - if cdMagnetlinkClock > magnetlinkClock { - err = m.persistence.UpdateMagnetlinkMessageClock(community.ID(), cdMagnetlinkClock) + if cdArchiveLinkClock > archiveLinkClock { + err = m.persistence.UpdateArchiveLinkMessageClock(community.ID(), cdArchiveLinkClock) if err != nil { return nil, err } @@ -3602,8 +3532,8 @@ func (m *Manager) SpectateCommunity(id types3.HexBytes) (*Community, error) { return community, nil } -func (m *Manager) GetMagnetlinkMessageClock(communityID types3.HexBytes) (uint64, error) { - return m.persistence.GetMagnetlinkMessageClock(communityID) +func (m *Manager) GetArchiveLinkMessageClock(communityID types3.HexBytes) (uint64, error) { + return m.persistence.GetArchiveLinkMessageClock(communityID) } func (m *Manager) GetCommunityRequestToJoinClock(pk *ecdsa.PublicKey, communityID string) (uint64, error) { @@ -3627,7 +3557,7 @@ func (m *Manager) GetRequestToJoinByPkAndCommunityID(pk *ecdsa.PublicKey, commun return m.persistence.GetRequestToJoinByPkAndCommunityID(crypto.PubkeyToHex(pk), communityID) } -func (m *Manager) UpdateCommunityDescriptionMagnetlinkMessageClock(communityID types3.HexBytes, clock uint64) error { +func (m *Manager) UpdateCommunityDescriptionArchiveLinkMessageClock(communityID types3.HexBytes, clock uint64) error { m.communityLock.Lock(communityID) defer m.communityLock.Unlock(communityID) @@ -3635,20 +3565,20 @@ func (m *Manager) UpdateCommunityDescriptionMagnetlinkMessageClock(communityID t if err != nil { return err } - community.config.CommunityDescription.ArchiveMagnetlinkClock = clock + community.config.CommunityDescription.ArchiveLinkClock = clock return m.SaveCommunity(community) } -func (m *Manager) UpdateMagnetlinkMessageClock(communityID types3.HexBytes, clock uint64) error { - return m.persistence.UpdateMagnetlinkMessageClock(communityID, clock) +func (m *Manager) UpdateArchiveLinkMessageClock(communityID types3.HexBytes, clock uint64) error { + return m.persistence.UpdateArchiveLinkMessageClock(communityID, clock) } -func (m *Manager) UpdateLastSeenMagnetlink(communityID types3.HexBytes, magnetlinkURI string) error { - return m.persistence.UpdateLastSeenMagnetlink(communityID, magnetlinkURI) +func (m *Manager) UpdateLastSeenArchiveLink(communityID types3.HexBytes, archiveLink string) error { + return m.persistence.UpdateLastSeenArchiveLink(communityID, archiveLink) } -func (m *Manager) GetLastSeenMagnetlink(communityID types3.HexBytes) (string, error) { - return m.persistence.GetLastSeenMagnetlink(communityID) +func (m *Manager) GetLastSeenArchiveLink(communityID types3.HexBytes) (string, error) { + return m.persistence.GetLastSeenArchiveLink(communityID) } func (m *Manager) LeaveCommunity(id types3.HexBytes) (*Community, error) { diff --git a/protocol/communities/manager_archive.go b/protocol/communities/manager_archive.go deleted file mode 100644 index c2c81b9c320..00000000000 --- a/protocol/communities/manager_archive.go +++ /dev/null @@ -1,739 +0,0 @@ -//go:build !disable_torrent -// +build !disable_torrent - -// Attribution to Pascal Precht, for further context please view the below issues -// - https://github.com/status-im/status-go/issues/2563 -// - https://github.com/status-im/status-go/issues/2565 -// - https://github.com/status-im/status-go/issues/2567 -// - https://github.com/status-im/status-go/issues/2568 - -package communities - -import ( - "crypto/ecdsa" - "errors" - "fmt" - "net" - "os" - "path" - "sort" - "sync" - "time" - - "github.com/status-im/status-go/common" - "github.com/status-im/status-go/internal/crypto/types" - "github.com/status-im/status-go/params" - "github.com/status-im/status-go/pkg/messaging" - types2 "github.com/status-im/status-go/pkg/messaging/types" - "github.com/status-im/status-go/signal" - - "github.com/anacrolix/torrent" - "github.com/anacrolix/torrent/metainfo" - "go.uber.org/zap" -) - -type archiveMDSlice []*archiveMetadata - -type archiveMetadata struct { - hash string - from uint64 -} - -func (md archiveMDSlice) Len() int { - return len(md) -} - -func (md archiveMDSlice) Swap(i, j int) { - md[i], md[j] = md[j], md[i] -} - -func (md archiveMDSlice) Less(i, j int) bool { - return md[i].from > md[j].from -} - -type EncodedArchiveData struct { - padding int - bytes []byte -} - -type ArchiveManager struct { - common.PauseBroadcaster - - torrentConfig *params.TorrentConfig - torrentClient *torrent.Client - torrentTasks map[string]metainfo.Hash - historyArchiveDownloadTasks map[string]*HistoryArchiveDownloadTask - historyArchiveTasksWaitGroup sync.WaitGroup - historyArchiveTasks sync.Map // stores `chan struct{}` - - logger *zap.Logger - persistence *Persistence - messaging *messaging.API - identity *ecdsa.PrivateKey - - *ArchiveFileManager - publisher Publisher -} - -// NewArchiveManager this function is only built and called when the "disable_torrent" build tag is not set -// In this case this version of NewArchiveManager will return the full Desktop ArchiveManager ensuring that the -// build command will import and build the torrent deps for the Desktop OSes. -// NOTE: It is intentional that this file contains the identical function name as in "manager_archive_nop.go" -func NewArchiveManager(amc *ArchiveManagerConfig) *ArchiveManager { - return &ArchiveManager{ - torrentConfig: amc.TorrentConfig, - torrentTasks: make(map[string]metainfo.Hash), - historyArchiveDownloadTasks: make(map[string]*HistoryArchiveDownloadTask), - - logger: amc.Logger, - persistence: amc.Persistence, - messaging: amc.Messaging, - identity: amc.Identity, - - publisher: amc.Publisher, - ArchiveFileManager: NewArchiveFileManager(amc), - } -} - -func (m *ArchiveManager) SetPaused(paused bool) { - if paused { - m.MarkPaused() - } else { - m.MarkResumed() - } -} - -func (m *ArchiveManager) SetOnline(online bool) { - if online { - if m.torrentConfig != nil && m.torrentConfig.Enabled && !m.torrentClientStarted() { - err := m.StartTorrentClient() - if err != nil { - m.logger.Error("couldn't start torrent client", zap.Error(err)) - } - } - } -} - -func (m *ArchiveManager) SetTorrentConfig(config *params.TorrentConfig) { - m.torrentConfig = config - m.ArchiveFileManager.torrentConfig = config -} - -// getTCPandUDPport will return the same port number given if != 0, -// otherwise, it will attempt to find a free random tcp and udp port using -// the same number for both protocols -func (m *ArchiveManager) getTCPandUDPport(portNumber int) (int, error) { - if portNumber != 0 { - return portNumber, nil - } - - // Find free port - for i := 0; i < 10; i++ { - port := func() int { - tcpAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort("localhost", "0")) - if err != nil { - m.logger.Warn("unable to resolve tcp addr: %v", zap.Error(err)) - return 0 - } - - tcpListener, err := net.ListenTCP("tcp", tcpAddr) - if err != nil { - m.logger.Warn("unable to listen on addr", zap.Stringer("addr", tcpAddr), zap.Error(err)) - return 0 - } - defer tcpListener.Close() - - port := tcpListener.Addr().(*net.TCPAddr).Port - - udpAddr, err := net.ResolveUDPAddr("udp", net.JoinHostPort("localhost", fmt.Sprintf("%d", port))) - if err != nil { - m.logger.Warn("unable to resolve udp addr: %v", zap.Error(err)) - return 0 - } - - udpListener, err := net.ListenUDP("udp", udpAddr) - if err != nil { - m.logger.Warn("unable to listen on addr", zap.Stringer("addr", udpAddr), zap.Error(err)) - return 0 - } - defer udpListener.Close() - - return port - }() - - if port != 0 { - return port, nil - } - } - - return 0, fmt.Errorf("no free port found") -} - -func (m *ArchiveManager) StartTorrentClient() error { - if m.torrentConfig == nil { - return fmt.Errorf("can't start torrent client: missing torrentConfig") - } - - if m.torrentClientStarted() { - return nil - } - - port, err := m.getTCPandUDPport(m.torrentConfig.Port) - if err != nil { - return err - } - - config := torrent.NewDefaultClientConfig() - config.SetListenAddr(":" + fmt.Sprint(port)) - config.Seed = true - - config.DataDir = m.torrentConfig.DataDir - - if _, err := os.Stat(m.torrentConfig.DataDir); os.IsNotExist(err) { - err := os.MkdirAll(m.torrentConfig.DataDir, 0700) - if err != nil { - return err - } - } - - m.logger.Info("Starting torrent client", zap.Any("port", port)) - // Instantiating the client will make it bootstrap and listen eagerly, - // so no go routine is needed here - client, err := torrent.NewClient(config) - if err != nil { - return err - } - m.torrentClient = client - return nil -} - -func (m *ArchiveManager) Stop() error { - if m.torrentClientStarted() { - m.stopHistoryArchiveTasksIntervals() - m.logger.Info("Stopping torrent client") - errs := m.torrentClient.Close() - if len(errs) > 0 { - return errors.Join(errs...) - } - m.torrentClient = nil - } - return nil -} - -func (m *ArchiveManager) torrentClientStarted() bool { - return m.torrentClient != nil -} - -func (m *ArchiveManager) IsReady() bool { - // Simply checking for `torrentConfig.Enabled` isn't enough - // as there's a possibility that the torrent client couldn't - // be instantiated (for example in case of port conflicts) - return m.torrentConfig != nil && - m.torrentConfig.Enabled && - m.torrentClientStarted() -} - -func (m *ArchiveManager) GetCommunityChatsFilters(communityID types.HexBytes) (types2.ChatFilters, error) { - chatIDs, err := m.persistence.GetCommunityChatIDs(communityID) - if err != nil { - return nil, err - } - - filters := types2.ChatFilters{} - for _, cid := range chatIDs { - filter := m.messaging.ChatFilterByChatID(cid) - if filter != nil { - filters = append(filters, filter) - } - } - return filters, nil -} - -func (m *ArchiveManager) GetCommunityChatsTopics(communityID types.HexBytes) ([]types2.ContentTopic, error) { - filters, err := m.GetCommunityChatsFilters(communityID) - if err != nil { - return nil, err - } - - topics := []types2.ContentTopic{} - for _, filter := range filters { - topics = append(topics, filter.ContentTopic()) - } - - return topics, nil -} - -func (m *ArchiveManager) getOldestWakuMessageTimestamp(topics []types2.ContentTopic) (uint64, error) { - return m.persistence.GetOldestWakuMessageTimestamp(topics) -} - -func (m *ArchiveManager) getLastMessageArchiveEndDate(communityID types.HexBytes) (uint64, error) { - return m.persistence.GetLastMessageArchiveEndDate(communityID) -} - -func (m *ArchiveManager) GetHistoryArchivePartitionStartTimestamp(communityID types.HexBytes) (uint64, error) { - filters, err := m.GetCommunityChatsFilters(communityID) - if err != nil { - m.logger.Error("failed to get community chats filters", zap.Error(err)) - return 0, err - } - - if len(filters) == 0 { - // If we don't have chat filters, we likely don't have any chats - // associated to this community, which means there's nothing more - // to do here - return 0, nil - } - - topics := []types2.ContentTopic{} - - for _, filter := range filters { - topics = append(topics, filter.ContentTopic()) - } - - lastArchiveEndDateTimestamp, err := m.getLastMessageArchiveEndDate(communityID) - if err != nil { - m.logger.Error("failed to get last archive end date", zap.Error(err)) - return 0, err - } - - if lastArchiveEndDateTimestamp == 0 { - // If we don't have a tracked last message archive end date, it - // means we haven't created an archive before, which means - // the next thing to look at is the oldest waku message timestamp for - // this community - lastArchiveEndDateTimestamp, err = m.getOldestWakuMessageTimestamp(topics) - if err != nil { - m.logger.Error("failed to get oldest waku message timestamp", zap.Error(err)) - return 0, err - } - if lastArchiveEndDateTimestamp == 0 { - // This means there's no waku message stored for this community so far - // (even after requesting possibly missed messages), so no messages exist yet that can be archived - m.logger.Debug("can't find valid `lastArchiveEndTimestamp`") - return 0, nil - } - } - - return lastArchiveEndDateTimestamp, nil -} - -func (m *ArchiveManager) CreateAndSeedHistoryArchive(communityID types.HexBytes, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) error { - m.UnseedHistoryArchiveTorrent(communityID) - _, err := m.ArchiveFileManager.CreateHistoryArchiveTorrentFromDB(communityID, topics, startDate, endDate, partition, encrypt) - if err != nil { - return err - } - return m.SeedHistoryArchiveTorrent(communityID) -} - -func (m *ArchiveManager) StartHistoryArchiveTasksInterval(community *Community, interval time.Duration) { - defer common.LogOnPanic() - id := community.IDString() - if _, exists := m.historyArchiveTasks.Load(id); exists { - m.logger.Error("history archive tasks interval already in progress", zap.String("id", id)) - return - } - - cancel := make(chan struct{}) - m.historyArchiveTasks.Store(id, cancel) - m.historyArchiveTasksWaitGroup.Add(1) - - ticker := time.NewTicker(interval) - defer ticker.Stop() - sub := m.Subscribe() - defer sub.Unsubscribe() - paused := <-sub.C() - var tickerC <-chan time.Time - if !paused { - tickerC = ticker.C - } - - m.logger.Debug("starting history archive tasks interval", zap.String("id", id)) - for { - select { - case pausedState, ok := <-sub.C(): - if !ok { - m.UnseedHistoryArchiveTorrent(community.ID()) - m.historyArchiveTasks.Delete(id) - m.historyArchiveTasksWaitGroup.Done() - return - } - paused = pausedState - if paused { - tickerC = nil - } else { - tickerC = ticker.C - } - case <-tickerC: - m.logger.Debug("starting archive task...", zap.String("id", id)) - lastArchiveEndDateTimestamp, err := m.GetHistoryArchivePartitionStartTimestamp(community.ID()) - if err != nil { - m.logger.Error("failed to get last archive end date", zap.Error(err)) - continue - } - - if lastArchiveEndDateTimestamp == 0 { - // This means there are no waku messages for this community, - // so nothing to do here - m.logger.Debug("couldn't determine archive start date - skipping") - continue - } - - topics, err := m.GetCommunityChatsTopics(community.ID()) - if err != nil { - m.logger.Error("failed to get community chat topics ", zap.Error(err)) - continue - } - filter := m.messaging.ChatFilterByChatID(community.UniversalChatID()) - if filter == nil { - m.logger.Error("failed to get chat filter", zap.String("community's UniversalChatID", community.UniversalChatID())) - continue - } - // adding the content-topic used for member updates. - // since member updates would not be too frequent i.e only addition/deletion would add a new message, - // this shouldn't cause too much increase in size of archive generated. - topics = append(topics, filter.ContentTopic()) - - ts := time.Now().Unix() - to := time.Unix(ts, 0) - lastArchiveEndDate := time.Unix(int64(lastArchiveEndDateTimestamp), 0) - - err = m.CreateAndSeedHistoryArchive(community.ID(), topics, lastArchiveEndDate, to, interval, community.Encrypted()) - if err != nil { - m.logger.Error("failed to create and seed history archive", zap.Error(err)) - continue - } - case <-cancel: - m.UnseedHistoryArchiveTorrent(community.ID()) - m.historyArchiveTasks.Delete(id) - m.historyArchiveTasksWaitGroup.Done() - return - } - } -} - -func (m *ArchiveManager) stopHistoryArchiveTasksIntervals() { - m.historyArchiveTasks.Range(func(_, task interface{}) bool { - close(task.(chan struct{})) // Need to cast to the chan - return true - }) - // Stoping archive interval tasks is async, so we need - // to wait for all of them to be closed before we shutdown - // the torrent client - m.historyArchiveTasksWaitGroup.Wait() -} - -func (m *ArchiveManager) StopHistoryArchiveTasksInterval(communityID types.HexBytes) { - task, exists := m.historyArchiveTasks.Load(communityID.String()) - if exists { - m.logger.Info("Stopping history archive tasks interval", zap.Any("id", communityID.String())) - close(task.(chan struct{})) // Need to cast to the chan - } -} - -func (m *ArchiveManager) SeedHistoryArchiveTorrent(communityID types.HexBytes) error { - m.UnseedHistoryArchiveTorrent(communityID) - - id := communityID.String() - torrentFile := torrentFile(m.torrentConfig.TorrentDir, id) - - metaInfo, err := metainfo.LoadFromFile(torrentFile) - if err != nil { - return err - } - - info, err := metaInfo.UnmarshalInfo() - if err != nil { - return err - } - - hash := metaInfo.HashInfoBytes() - m.torrentTasks[id] = hash - - if err != nil { - return err - } - - torrent, err := m.torrentClient.AddTorrent(metaInfo) - if err != nil { - return err - } - - torrent.DownloadAll() - - m.publisher.publish(&Subscription{ - HistoryArchivesSeedingSignal: &signal.HistoryArchivesSeedingSignal{ - CommunityID: communityID.String(), - }, - }) - - magnetLink := metaInfo.Magnet(nil, &info).String() - - m.logger.Debug("seeding torrent", zap.String("id", id), zap.String("magnetLink", magnetLink)) - return nil -} - -func (m *ArchiveManager) UnseedHistoryArchiveTorrent(communityID types.HexBytes) { - id := communityID.String() - - hash, exists := m.torrentTasks[id] - - if exists { - torrent, ok := m.torrentClient.Torrent(hash) - if ok { - m.logger.Debug("Unseeding and dropping torrent for community: ", zap.Any("id", id)) - torrent.Drop() - delete(m.torrentTasks, id) - - m.publisher.publish(&Subscription{ - HistoryArchivesUnseededSignal: &signal.HistoryArchivesUnseededSignal{ - CommunityID: id, - }, - }) - } - } -} - -func (m *ArchiveManager) IsSeedingHistoryArchiveTorrent(communityID types.HexBytes) bool { - id := communityID.String() - hash := m.torrentTasks[id] - torrent, ok := m.torrentClient.Torrent(hash) - return ok && torrent.Seeding() -} - -func (m *ArchiveManager) GetHistoryArchiveDownloadTask(communityID string) *HistoryArchiveDownloadTask { - return m.historyArchiveDownloadTasks[communityID] -} - -func (m *ArchiveManager) AddHistoryArchiveDownloadTask(communityID string, task *HistoryArchiveDownloadTask) { - m.historyArchiveDownloadTasks[communityID] = task -} - -func (m *ArchiveManager) DownloadHistoryArchivesByMagnetlink(communityID types.HexBytes, magnetlink string, cancelTask chan struct{}) (*HistoryArchiveDownloadTaskInfo, error) { - - id := communityID.String() - - ml, err := metainfo.ParseMagnetUri(magnetlink) - if err != nil { - return nil, err - } - - m.logger.Debug("adding torrent via magnetlink for community", zap.String("id", id), zap.String("magnetlink", magnetlink)) - torrent, err := m.torrentClient.AddMagnet(magnetlink) - if err != nil { - return nil, err - } - - downloadTaskInfo := &HistoryArchiveDownloadTaskInfo{ - TotalDownloadedArchivesCount: 0, - TotalArchivesCount: 0, - Cancelled: false, - } - - m.torrentTasks[id] = ml.InfoHash - timeout := time.After(20 * time.Second) - - m.logger.Debug("fetching torrent info", zap.String("magnetlink", magnetlink)) - select { - case <-timeout: - return nil, ErrTorrentTimedout - case <-cancelTask: - m.logger.Debug("cancelled fetching torrent info") - downloadTaskInfo.Cancelled = true - return downloadTaskInfo, nil - case <-torrent.GotInfo(): - - files := torrent.Files() - - i, ok := findIndexFile(files) - if !ok { - // We're dealing with a malformed torrent, so don't do anything - return nil, errors.New("malformed torrent data") - } - - indexFile := files[i] - indexFile.Download() - - m.logger.Debug("downloading history archive index") - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - sub := m.Subscribe() - defer sub.Unsubscribe() - paused := <-sub.C() - var tickerC <-chan time.Time - if !paused { - tickerC = ticker.C - } - - for { - select { - case pausedState, ok := <-sub.C(): - if !ok { - return nil, errors.New("lifecycle subscription closed") - } - paused = pausedState - if paused { - tickerC = nil - } else { - tickerC = ticker.C - } - case <-cancelTask: - m.logger.Debug("cancelled downloading archive index") - downloadTaskInfo.Cancelled = true - return downloadTaskInfo, nil - case <-tickerC: - if indexFile.BytesCompleted() == indexFile.Length() { - - index, err := m.ArchiveFileManager.LoadHistoryArchiveIndexFromFile(m.identity, communityID) - if err != nil { - return nil, err - } - - existingArchiveIDs, err := m.persistence.GetDownloadedMessageArchiveIDs(communityID) - if err != nil { - return nil, err - } - - if len(existingArchiveIDs) == len(index.Archives) { - m.logger.Debug("download cancelled, no new archives") - return downloadTaskInfo, nil - } - - downloadTaskInfo.TotalDownloadedArchivesCount = len(existingArchiveIDs) - downloadTaskInfo.TotalArchivesCount = len(index.Archives) - - archiveHashes := make(archiveMDSlice, 0, downloadTaskInfo.TotalArchivesCount) - - for hash, metadata := range index.Archives { - archiveHashes = append(archiveHashes, &archiveMetadata{hash: hash, from: metadata.Metadata.From}) - } - - sort.Sort(sort.Reverse(archiveHashes)) - - m.publisher.publish(&Subscription{ - DownloadingHistoryArchivesStartedSignal: &signal.DownloadingHistoryArchivesStartedSignal{ - CommunityID: communityID.String(), - }, - }) - - for _, hd := range archiveHashes { - - hash := hd.hash - hasArchive := false - - for _, existingHash := range existingArchiveIDs { - if existingHash == hash { - hasArchive = true - break - } - } - if hasArchive { - continue - } - - metadata := index.Archives[hash] - startIndex := int(metadata.Offset) / pieceLength - endIndex := startIndex + int(metadata.Size)/pieceLength - - downloadMsg := fmt.Sprintf("downloading data for message archive (%d/%d)", downloadTaskInfo.TotalDownloadedArchivesCount+1, downloadTaskInfo.TotalArchivesCount) - m.logger.Debug(downloadMsg, zap.String("hash", hash)) - m.logger.Debug("pieces (start, end)", zap.Any("startIndex", startIndex), zap.Any("endIndex", endIndex-1)) - torrent.DownloadPieces(startIndex, endIndex) - - piecesCompleted := make(map[int]bool) - for i = startIndex; i < endIndex; i++ { - piecesCompleted[i] = false - } - - psc := torrent.SubscribePieceStateChanges() - downloadTicker := time.NewTicker(1 * time.Second) - defer downloadTicker.Stop() - var downloadTickerC <-chan time.Time - if !paused { - downloadTickerC = downloadTicker.C - } - - downloadLoop: - for { - select { - case pausedState, ok := <-sub.C(): - if !ok { - return nil, errors.New("lifecycle subscription closed") - } - paused = pausedState - if paused { - downloadTickerC = nil - } else { - downloadTickerC = downloadTicker.C - } - case <-downloadTickerC: - done := true - for i = startIndex; i < endIndex; i++ { - piecesCompleted[i] = torrent.PieceState(i).Complete - if !piecesCompleted[i] { - done = false - } - } - if done { - psc.Close() - break downloadLoop - } - case <-cancelTask: - m.logger.Debug("downloading archive data interrupted") - downloadTaskInfo.Cancelled = true - return downloadTaskInfo, nil - } - } - downloadTaskInfo.TotalDownloadedArchivesCount++ - err = m.persistence.SaveMessageArchiveID(communityID, hash) - if err != nil { - m.logger.Error("couldn't save message archive ID", zap.Error(err)) - continue - } - m.publisher.publish(&Subscription{ - HistoryArchiveDownloadedSignal: &signal.HistoryArchiveDownloadedSignal{ - CommunityID: communityID.String(), - From: int(metadata.Metadata.From), - To: int(metadata.Metadata.To), - }, - }) - } - m.publisher.publish(&Subscription{ - HistoryArchivesSeedingSignal: &signal.HistoryArchivesSeedingSignal{ - CommunityID: communityID.String(), - }, - }) - m.logger.Debug("finished downloading archives") - return downloadTaskInfo, nil - } - } - } - } -} - -func (m *ArchiveManager) TorrentFileExists(communityID string) bool { - _, err := os.Stat(torrentFile(m.torrentConfig.TorrentDir, communityID)) - return err == nil -} - -func topicsAsByteArrays(topics []types2.ContentTopic) [][]byte { - var topicsAsByteArrays [][]byte - for _, t := range topics { - topicsAsByteArrays = append(topicsAsByteArrays, t.Bytes()) - } - return topicsAsByteArrays -} - -func findIndexFile(files []*torrent.File) (index int, ok bool) { - for i, f := range files { - if f.DisplayPath() == "index" { - return i, true - } - } - return 0, false -} - -func torrentFile(torrentDir, communityID string) string { - return path.Join(torrentDir, communityID+".torrent") -} diff --git a/protocol/communities/manager_archive_file.go b/protocol/communities/manager_archive_file.go deleted file mode 100644 index a19e00966e6..00000000000 --- a/protocol/communities/manager_archive_file.go +++ /dev/null @@ -1,495 +0,0 @@ -//go:build !disable_torrent -// +build !disable_torrent - -// Attribution to Pascal Precht, for further context please view the below issues -// - https://github.com/status-im/status-go/issues/2563 -// - https://github.com/status-im/status-go/issues/2565 -// - https://github.com/status-im/status-go/issues/2567 -// - https://github.com/status-im/status-go/issues/2568 - -package communities - -import ( - "crypto/ecdsa" - "os" - "path" - "time" - - "github.com/status-im/status-go/internal/crypto" - "github.com/status-im/status-go/internal/crypto/types" - "github.com/status-im/status-go/params" - "github.com/status-im/status-go/pkg/messaging" - types2 "github.com/status-im/status-go/pkg/messaging/types" - "github.com/status-im/status-go/protocol/protobuf" - "github.com/status-im/status-go/signal" - - "github.com/anacrolix/torrent/bencode" - "github.com/anacrolix/torrent/metainfo" - "github.com/golang/protobuf/proto" - "go.uber.org/zap" -) - -type ArchiveFileManager struct { - torrentConfig *params.TorrentConfig - - logger *zap.Logger - persistence *Persistence - identity *ecdsa.PrivateKey - messaging *messaging.API - - publisher Publisher -} - -func NewArchiveFileManager(amc *ArchiveManagerConfig) *ArchiveFileManager { - return &ArchiveFileManager{ - torrentConfig: amc.TorrentConfig, - logger: amc.Logger, - persistence: amc.Persistence, - identity: amc.Identity, - messaging: amc.Messaging, - publisher: amc.Publisher, - } -} - -func (m *ArchiveFileManager) createHistoryArchiveTorrent(communityID types.HexBytes, msgs []*types2.ReceivedMessage, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { - - loadFromDB := len(msgs) == 0 - - from := startDate - to := from.Add(partition) - if to.After(endDate) { - to = endDate - } - - archiveDir := m.torrentConfig.DataDir + "/" + communityID.String() - torrentDir := m.torrentConfig.TorrentDir - indexPath := archiveDir + "/index" - dataPath := archiveDir + "/data" - - wakuMessageArchiveIndexProto := &protobuf.WakuMessageArchiveIndex{} - wakuMessageArchiveIndex := make(map[string]*protobuf.WakuMessageArchiveIndexMetadata) - archiveIDs := make([]string, 0) - - if _, err := os.Stat(archiveDir); os.IsNotExist(err) { - err := os.MkdirAll(archiveDir, 0700) - if err != nil { - return archiveIDs, err - } - } - if _, err := os.Stat(torrentDir); os.IsNotExist(err) { - err := os.MkdirAll(torrentDir, 0700) - if err != nil { - return archiveIDs, err - } - } - - _, err := os.Stat(indexPath) - if err == nil { - wakuMessageArchiveIndexProto, err = m.LoadHistoryArchiveIndexFromFile(m.identity, communityID) - if err != nil { - return archiveIDs, err - } - } - - var offset uint64 = 0 - - for hash, metadata := range wakuMessageArchiveIndexProto.Archives { - offset = offset + metadata.Size - wakuMessageArchiveIndex[hash] = metadata - } - - var encodedArchives []*EncodedArchiveData - topicsAsByteArrays := topicsAsByteArrays(topics) - - m.publisher.publish(&Subscription{CreatingHistoryArchivesSignal: &signal.CreatingHistoryArchivesSignal{ - CommunityID: communityID.String(), - }}) - - m.logger.Debug("creating archives", - zap.Any("startDate", startDate), - zap.Any("endDate", endDate), - zap.Duration("partition", partition), - ) - for { - if from.Equal(endDate) || from.After(endDate) { - break - } - m.logger.Debug("creating message archive", - zap.Any("from", from), - zap.Any("to", to), - ) - - var messages []types2.ReceivedMessage - if loadFromDB { - messages, err = m.persistence.GetWakuMessagesByFilterTopic(topics, uint64(from.Unix()), uint64(to.Unix())) - if err != nil { - return archiveIDs, err - } - } else { - for _, msg := range msgs { - if int64(msg.Timestamp) >= from.Unix() && int64(msg.Timestamp) < to.Unix() { - messages = append(messages, *msg) - } - } - } - - if len(messages) == 0 { - // No need to create an archive with zero messages - m.logger.Debug("no messages in this partition") - from = to - to = to.Add(partition) - if to.After(endDate) { - to = endDate - } - continue - } - - m.logger.Debug("creating archive with messages", zap.Int("messagesCount", len(messages))) - - // Not only do we partition messages, we also chunk them - // roughly by size, such that each chunk will not exceed a given - // size and archive data doesn't get too big - messageChunks := make([][]types2.ReceivedMessage, 0) - currentChunkSize := 0 - currentChunk := make([]types2.ReceivedMessage, 0) - - for _, msg := range messages { - msgSize := len(msg.Payload) + len(msg.Sig) - if msgSize > maxArchiveSizeInBytes { - // we drop messages this big - continue - } - - if currentChunkSize+msgSize > maxArchiveSizeInBytes { - messageChunks = append(messageChunks, currentChunk) - currentChunk = make([]types2.ReceivedMessage, 0) - currentChunkSize = 0 - } - currentChunk = append(currentChunk, msg) - currentChunkSize = currentChunkSize + msgSize - } - messageChunks = append(messageChunks, currentChunk) - - for _, messages := range messageChunks { - wakuMessageArchive := m.createWakuMessageArchive(from, to, messages, topicsAsByteArrays) - encodedArchive, err := proto.Marshal(wakuMessageArchive) - if err != nil { - return archiveIDs, err - } - - if encrypt { - encodedArchive, err = m.messaging.BuildHashRatchetMessage(communityID, encodedArchive) - if err != nil { - return archiveIDs, err - } - } - - rawSize := len(encodedArchive) - padding := 0 - size := 0 - - if rawSize > pieceLength { - size = rawSize + pieceLength - (rawSize % pieceLength) - padding = size - rawSize - } else { - padding = pieceLength - rawSize - size = rawSize + padding - } - - wakuMessageArchiveIndexMetadata := &protobuf.WakuMessageArchiveIndexMetadata{ - Metadata: wakuMessageArchive.Metadata, - Offset: offset, - Size: uint64(size), - Padding: uint64(padding), - } - - wakuMessageArchiveIndexMetadataBytes, err := proto.Marshal(wakuMessageArchiveIndexMetadata) - if err != nil { - return archiveIDs, err - } - - archiveID := crypto.Keccak256Hash(wakuMessageArchiveIndexMetadataBytes).String() - archiveIDs = append(archiveIDs, archiveID) - wakuMessageArchiveIndex[archiveID] = wakuMessageArchiveIndexMetadata - encodedArchives = append(encodedArchives, &EncodedArchiveData{bytes: encodedArchive, padding: padding}) - offset = offset + uint64(rawSize) + uint64(padding) - } - - from = to - to = to.Add(partition) - if to.After(endDate) { - to = endDate - } - } - - if len(encodedArchives) > 0 { - - dataBytes := make([]byte, 0) - - for _, encodedArchiveData := range encodedArchives { - dataBytes = append(dataBytes, encodedArchiveData.bytes...) - dataBytes = append(dataBytes, make([]byte, encodedArchiveData.padding)...) - } - - wakuMessageArchiveIndexProto.Archives = wakuMessageArchiveIndex - indexBytes, err := proto.Marshal(wakuMessageArchiveIndexProto) - if err != nil { - return archiveIDs, err - } - - if encrypt { - indexBytes, err = m.messaging.BuildHashRatchetMessage(communityID, indexBytes) - if err != nil { - return archiveIDs, err - } - } - - err = os.WriteFile(indexPath, indexBytes, 0644) // nolint: gosec - if err != nil { - return archiveIDs, err - } - - file, err := os.OpenFile(dataPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - return archiveIDs, err - } - defer file.Close() - - _, err = file.Write(dataBytes) - if err != nil { - return archiveIDs, err - } - - metaInfo := metainfo.MetaInfo{ - AnnounceList: defaultAnnounceList, - } - metaInfo.SetDefaults() - metaInfo.CreatedBy = crypto.PubkeyToHex(&m.identity.PublicKey) - - info := metainfo.Info{ - PieceLength: int64(pieceLength), - } - - err = info.BuildFromFilePath(archiveDir) - if err != nil { - return archiveIDs, err - } - - metaInfo.InfoBytes, err = bencode.Marshal(info) - if err != nil { - return archiveIDs, err - } - - metaInfoBytes, err := bencode.Marshal(metaInfo) - if err != nil { - return archiveIDs, err - } - - err = os.WriteFile(torrentFile(m.torrentConfig.TorrentDir, communityID.String()), metaInfoBytes, 0644) // nolint: gosec - if err != nil { - return archiveIDs, err - } - - m.logger.Debug("torrent created", zap.Any("from", startDate.Unix()), zap.Any("to", endDate.Unix())) - - m.publisher.publish(&Subscription{ - HistoryArchivesCreatedSignal: &signal.HistoryArchivesCreatedSignal{ - CommunityID: communityID.String(), - From: int(startDate.Unix()), - To: int(endDate.Unix()), - }, - }) - } else { - m.logger.Debug("no archives created") - m.publisher.publish(&Subscription{ - NoHistoryArchivesCreatedSignal: &signal.NoHistoryArchivesCreatedSignal{ - CommunityID: communityID.String(), - From: int(startDate.Unix()), - To: int(endDate.Unix()), - }, - }) - } - - lastMessageArchiveEndDate, err := m.persistence.GetLastMessageArchiveEndDate(communityID) - if err != nil { - return archiveIDs, err - } - - if lastMessageArchiveEndDate > 0 { - err = m.persistence.UpdateLastMessageArchiveEndDate(communityID, uint64(from.Unix())) - } else { - err = m.persistence.SaveLastMessageArchiveEndDate(communityID, uint64(from.Unix())) - } - if err != nil { - return archiveIDs, err - } - return archiveIDs, nil -} - -func (m *ArchiveFileManager) archiveIndexFile(communityID string) string { - return path.Join(m.torrentConfig.DataDir, communityID, "index") -} - -func (m *ArchiveFileManager) createWakuMessageArchive(from time.Time, to time.Time, messages []types2.ReceivedMessage, topics [][]byte) *protobuf.WakuMessageArchive { - var wakuMessages []*protobuf.WakuMessage - - for _, msg := range messages { - wakuMessage := &protobuf.WakuMessage{ - Sig: msg.Sig, - Timestamp: uint64(msg.Timestamp), - Topic: msg.Topic.Bytes(), - Payload: msg.Payload, - Padding: msg.Padding, - Hash: msg.Hash, - ThirdPartyId: msg.ThirdPartyID, - } - wakuMessages = append(wakuMessages, wakuMessage) - } - - metadata := protobuf.WakuMessageArchiveMetadata{ - From: uint64(from.Unix()), - To: uint64(to.Unix()), - ContentTopic: topics, - } - - wakuMessageArchive := &protobuf.WakuMessageArchive{ - Metadata: &metadata, - Messages: wakuMessages, - } - return wakuMessageArchive -} - -func (m *ArchiveFileManager) CreateHistoryArchiveTorrentFromMessages(communityID types.HexBytes, messages []*types2.ReceivedMessage, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { - return m.createHistoryArchiveTorrent(communityID, messages, topics, startDate, endDate, partition, encrypt) -} - -func (m *ArchiveFileManager) CreateHistoryArchiveTorrentFromDB(communityID types.HexBytes, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { - return m.createHistoryArchiveTorrent(communityID, make([]*types2.ReceivedMessage, 0), topics, startDate, endDate, partition, encrypt) -} - -func (m *ArchiveFileManager) GetMessageArchiveIDsToImport(communityID types.HexBytes) ([]string, error) { - return m.persistence.GetMessageArchiveIDsToImport(communityID) -} - -func (m *ArchiveFileManager) SaveMessageArchiveID(communityID types.HexBytes, hash string) error { - return m.persistence.SaveMessageArchiveID(communityID, hash) -} - -func (m *ArchiveFileManager) SetMessageArchiveIDImported(communityID types.HexBytes, hash string, imported bool) error { - return m.persistence.SetMessageArchiveIDImported(communityID, hash, imported) -} - -func (m *ArchiveFileManager) GetHistoryArchiveMagnetlink(communityID types.HexBytes) (string, error) { - id := communityID.String() - torrentFile := torrentFile(m.torrentConfig.TorrentDir, id) - - metaInfo, err := metainfo.LoadFromFile(torrentFile) - if err != nil { - return "", err - } - - info, err := metaInfo.UnmarshalInfo() - if err != nil { - return "", err - } - - return metaInfo.Magnet(nil, &info).String(), nil -} - -func (m *ArchiveFileManager) archiveDataFile(communityID string) string { - return path.Join(m.torrentConfig.DataDir, communityID, "data") -} - -func (m *ArchiveFileManager) ExtractMessagesFromHistoryArchive(communityID types.HexBytes, archiveID string) ([]*protobuf.WakuMessage, error) { - id := communityID.String() - - index, err := m.LoadHistoryArchiveIndexFromFile(m.identity, communityID) - if err != nil { - return nil, err - } - - dataFile, err := os.Open(m.archiveDataFile(id)) - if err != nil { - return nil, err - } - defer dataFile.Close() - - m.logger.Debug("extracting messages from history archive", - zap.String("communityID", communityID.String()), - zap.String("archiveID", archiveID)) - metadata := index.Archives[archiveID] - - _, err = dataFile.Seek(int64(metadata.Offset), 0) - if err != nil { - m.logger.Error("failed to seek archive data file", zap.Error(err)) - return nil, err - } - - data := make([]byte, metadata.Size-metadata.Padding) - m.logger.Debug("loading history archive data into memory", zap.Float64("data_size_MB", float64(metadata.Size-metadata.Padding)/1024.0/1024.0)) - _, err = dataFile.Read(data) - if err != nil { - m.logger.Error("failed failed to read archive data", zap.Error(err)) - return nil, err - } - - archive := &protobuf.WakuMessageArchive{} - - err = proto.Unmarshal(data, archive) - if err != nil { - pk, err := crypto.DecompressPubkey(communityID) - if err != nil { - m.logger.Error("failed to decompress community pubkey", zap.Error(err)) - return nil, err - } - - decryptedData, err := m.messaging.DecryptMessage(m.identity, pk, data) - if err != nil { - m.logger.Error("failed to decrypt message archive", zap.Error(err)) - return nil, err - } - - err = proto.Unmarshal(decryptedData, archive) - if err != nil { - m.logger.Error("failed to unmarshal message archive", zap.Error(err)) - return nil, err - } - } - return archive.Messages, nil -} - -func (m *ArchiveFileManager) LoadHistoryArchiveIndexFromFile(myKey *ecdsa.PrivateKey, communityID types.HexBytes) (*protobuf.WakuMessageArchiveIndex, error) { - wakuMessageArchiveIndexProto := &protobuf.WakuMessageArchiveIndex{} - - indexPath := m.archiveIndexFile(communityID.String()) - indexData, err := os.ReadFile(indexPath) - if err != nil { - return nil, err - } - - err = proto.Unmarshal(indexData, wakuMessageArchiveIndexProto) - if err != nil { - return nil, err - } - - if len(wakuMessageArchiveIndexProto.Archives) == 0 && len(indexData) > 0 { - // This means we're dealing with an encrypted index file, so we have to decrypt it first - pk, err := crypto.DecompressPubkey(communityID) - if err != nil { - return nil, err - } - - decryptedData, err := m.messaging.DecryptMessage(myKey, pk, indexData) - if err != nil { - m.logger.Error("failed to decrypt message archive", zap.Error(err)) - return nil, err - } - - err = proto.Unmarshal(decryptedData, wakuMessageArchiveIndexProto) - if err != nil { - return nil, err - } - } - - return wakuMessageArchiveIndexProto, nil -} diff --git a/protocol/communities/manager_archive_file_nop.go b/protocol/communities/manager_archive_file_nop.go deleted file mode 100644 index 8aaaccf05f5..00000000000 --- a/protocol/communities/manager_archive_file_nop.go +++ /dev/null @@ -1,47 +0,0 @@ -//go:build disable_torrent -// +build disable_torrent - -package communities - -import ( - "crypto/ecdsa" - "time" - - "github.com/status-im/status-go/internal/crypto/types" - types2 "github.com/status-im/status-go/pkg/messaging/types" - "github.com/status-im/status-go/protocol/protobuf" -) - -type ArchiveFileManagerNop struct{} - -func (amm *ArchiveFileManagerNop) CreateHistoryArchiveTorrentFromMessages(communityID types.HexBytes, messages []*types2.ReceivedMessage, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { - return nil, nil -} - -func (amm *ArchiveFileManagerNop) CreateHistoryArchiveTorrentFromDB(communityID types.HexBytes, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) ([]string, error) { - return nil, nil -} - -func (amm *ArchiveFileManagerNop) SaveMessageArchiveID(communityID types.HexBytes, hash string) error { - return nil -} - -func (amm *ArchiveFileManagerNop) GetMessageArchiveIDsToImport(communityID types.HexBytes) ([]string, error) { - return nil, nil -} - -func (amm *ArchiveFileManagerNop) SetMessageArchiveIDImported(communityID types.HexBytes, hash string, imported bool) error { - return nil -} - -func (amm *ArchiveFileManagerNop) ExtractMessagesFromHistoryArchive(communityID types.HexBytes, archiveID string) ([]*protobuf.WakuMessage, error) { - return nil, nil -} - -func (amm *ArchiveFileManagerNop) GetHistoryArchiveMagnetlink(communityID types.HexBytes) (string, error) { - return "", nil -} - -func (amm *ArchiveFileManagerNop) LoadHistoryArchiveIndexFromFile(myKey *ecdsa.PrivateKey, communityID types.HexBytes) (*protobuf.WakuMessageArchiveIndex, error) { - return nil, nil -} diff --git a/protocol/communities/manager_archive_nop.go b/protocol/communities/manager_archive_nop.go deleted file mode 100644 index df332d30c35..00000000000 --- a/protocol/communities/manager_archive_nop.go +++ /dev/null @@ -1,90 +0,0 @@ -//go:build disable_torrent -// +build disable_torrent - -package communities - -import ( - "time" - - "github.com/status-im/status-go/internal/crypto/types" - "github.com/status-im/status-go/params" - types2 "github.com/status-im/status-go/pkg/messaging/types" -) - -type ArchiveManagerNop struct { - *ArchiveFileManagerNop -} - -// NewArchiveManager this function is only built and called when the "disable_torrent" build tag is set -// In this case this version of NewArchiveManager will return the mobile "nil" ArchiveManagerNop ensuring that the -// build command will not import or build the torrent deps for the mobile OS. -// NOTE: It is intentional that this file contains the identical function name as in "manager_archive.go" -func NewArchiveManager(amc *ArchiveManagerConfig) *ArchiveManagerNop { - return &ArchiveManagerNop{ - &ArchiveFileManagerNop{}, - } -} - -func (tmm *ArchiveManagerNop) SetPaused(paused bool) {} - -func (tmm *ArchiveManagerNop) SetOnline(online bool) {} - -func (tmm *ArchiveManagerNop) SetTorrentConfig(*params.TorrentConfig) {} - -func (tmm *ArchiveManagerNop) StartTorrentClient() error { - return nil -} - -func (tmm *ArchiveManagerNop) Stop() error { - return nil -} - -func (tmm *ArchiveManagerNop) IsReady() bool { - return false -} - -func (tmm *ArchiveManagerNop) GetCommunityChatsFilters(communityID types.HexBytes) (types2.ChatFilters, error) { - return nil, nil -} - -func (tmm *ArchiveManagerNop) GetCommunityChatsTopics(communityID types.HexBytes) ([]types2.ContentTopic, error) { - return nil, nil -} - -func (tmm *ArchiveManagerNop) GetHistoryArchivePartitionStartTimestamp(communityID types.HexBytes) (uint64, error) { - return 0, nil -} - -func (tmm *ArchiveManagerNop) CreateAndSeedHistoryArchive(communityID types.HexBytes, topics []types2.ContentTopic, startDate time.Time, endDate time.Time, partition time.Duration, encrypt bool) error { - return nil -} - -func (tmm *ArchiveManagerNop) StartHistoryArchiveTasksInterval(community *Community, interval time.Duration) { -} - -func (tmm *ArchiveManagerNop) StopHistoryArchiveTasksInterval(communityID types.HexBytes) {} - -func (tmm *ArchiveManagerNop) SeedHistoryArchiveTorrent(communityID types.HexBytes) error { - return nil -} - -func (tmm *ArchiveManagerNop) UnseedHistoryArchiveTorrent(communityID types.HexBytes) {} - -func (tmm *ArchiveManagerNop) IsSeedingHistoryArchiveTorrent(communityID types.HexBytes) bool { - return false -} - -func (tmm *ArchiveManagerNop) GetHistoryArchiveDownloadTask(communityID string) *HistoryArchiveDownloadTask { - return nil -} - -func (tmm *ArchiveManagerNop) AddHistoryArchiveDownloadTask(communityID string, task *HistoryArchiveDownloadTask) { -} - -func (tmm *ArchiveManagerNop) DownloadHistoryArchivesByMagnetlink(communityID types.HexBytes, magnetlink string, cancelTask chan struct{}) (*HistoryArchiveDownloadTaskInfo, error) { - return nil, nil -} - -func (tmm *ArchiveManagerNop) TorrentFileExists(communityID string) bool { - return false -} diff --git a/protocol/communities/manager_test.go b/protocol/communities/manager_test.go index 01d845ded53..46ab6606fa2 100644 --- a/protocol/communities/manager_test.go +++ b/protocol/communities/manager_test.go @@ -15,9 +15,9 @@ import ( gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/golang/protobuf/proto" _ "github.com/mutecomm/go-sqlcipher/v4" // require go-sqlcipher that overrides default implementation "github.com/stretchr/testify/suite" + "google.golang.org/protobuf/proto" "github.com/status-im/status-go/internal/crypto" "github.com/status-im/status-go/internal/db/appdatabase" @@ -25,8 +25,9 @@ import ( testutils2 "github.com/status-im/status-go/internal/testutils" "github.com/status-im/status-go/internal/testutils/fake" "github.com/status-im/status-go/params" - "github.com/status-im/status-go/pkg/messaging" "github.com/status-im/status-go/pkg/messaging/types" + "github.com/status-im/status-go/protocol/communities/archive" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" community_token "github.com/status-im/status-go/protocol/communities/token" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/requests" @@ -45,10 +46,19 @@ func TestManagerSuite(t *testing.T) { type ManagerSuite struct { suite.Suite manager *Manager - archiveManager *ArchiveManager + archiveManager archive.ArchiveService } -func (s *ManagerSuite) buildManagers(ownerVerifier OwnerVerifier) (*Manager, *ArchiveManager) { +func buildTorrentConfig() *params.TorrentConfig { + return ¶ms.TorrentConfig{ + Enabled: true, + DataDir: os.TempDir() + "/archivedata", + TorrentDir: os.TempDir() + "/torrents", + Port: 0, + } +} + +func (s *ManagerSuite) buildManagers(ownerVerifier OwnerVerifier) (*Manager, archive.ArchiveService) { db, err := testutils2.SetupTestMemorySQLDB(appdatabase.DbInitializer{}) s.Require().NoError(err, "creating sqlite db instance") err = sqlite.Migrate(db) @@ -63,7 +73,7 @@ func (s *ManagerSuite) buildManagers(ownerVerifier OwnerVerifier) (*Manager, *Ar s.Require().NoError(err) s.Require().NoError(m.Start()) - amc := &ArchiveManagerConfig{ + amc := &archivetypes.ArchiveManagerConfig{ TorrentConfig: buildTorrentConfig(), Logger: logger, Persistence: m.GetPersistence(), @@ -71,7 +81,7 @@ func (s *ManagerSuite) buildManagers(ownerVerifier OwnerVerifier) (*Manager, *Ar Identity: key, Publisher: m, } - t := NewArchiveManager(amc) + t := archive.NewArchiveManager(amc) s.Require().NoError(err) return m, t @@ -99,16 +109,6 @@ func tokenBalance(tokenID uint64, balance uint64) thirdparty.TokenBalance { } } -func (s *ManagerSuite) getHistoryTasksCount() int { - // sync.Map doesn't have a Len function, so we need to count manually - count := 0 - s.archiveManager.historyArchiveTasks.Range(func(_, _ interface{}) bool { - count++ - return true - }) - return count -} - type testCollectiblesManager struct { response map[uint64]map[gethcommon.Address]thirdparty.TokenBalancesPerContractAddress } @@ -479,505 +479,6 @@ func (s *ManagerSuite) TestGetControlledCommunitiesChatIDs() { s.Require().Len(controlledChatIDs, 1) } -func (s *ManagerSuite) TestStartAndStopTorrentClient() { - err := s.archiveManager.StartTorrentClient() - s.Require().NoError(err) - s.Require().NotNil(s.archiveManager.torrentClient) - defer s.archiveManager.Stop() //nolint: errcheck - - _, err = os.Stat(s.archiveManager.torrentConfig.DataDir) - s.Require().NoError(err) - s.Require().Equal(s.archiveManager.torrentClientStarted(), true) -} - -func (s *ManagerSuite) TestStartHistoryArchiveTasksInterval() { - err := s.archiveManager.StartTorrentClient() - s.Require().NoError(err) - defer s.archiveManager.Stop() //nolint: errcheck - - community, _, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - interval := 10 * time.Second - go s.archiveManager.StartHistoryArchiveTasksInterval(community, interval) - // Due to async exec we need to wait a bit until we check - // the task count. - time.Sleep(5 * time.Second) - - count := s.getHistoryTasksCount() - s.Require().Equal(count, 1) - - // We wait another 5 seconds to ensure the first tick has kicked in - time.Sleep(5 * time.Second) - - _, err = os.Stat(torrentFile(s.archiveManager.torrentConfig.TorrentDir, community.IDString())) - s.Require().Error(err) - - s.archiveManager.StopHistoryArchiveTasksInterval(community.ID()) - s.archiveManager.historyArchiveTasksWaitGroup.Wait() - count = s.getHistoryTasksCount() - s.Require().Equal(count, 0) -} - -func (s *ManagerSuite) TestStartHistoryArchiveTasksInterval_RespectsPausedLifecycle() { - err := s.archiveManager.StartTorrentClient() - s.Require().NoError(err) - defer s.archiveManager.Stop() //nolint: errcheck - - s.archiveManager.SetPaused(true) - defer s.archiveManager.SetPaused(false) - - community, _, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - interval := 300 * time.Millisecond - go s.archiveManager.StartHistoryArchiveTasksInterval(community, interval) - - time.Sleep(200 * time.Millisecond) - s.Require().Equal(1, s.getHistoryTasksCount()) - - // Unpause to exercise lifecycle transition handling in the running loop. - s.archiveManager.SetPaused(false) - time.Sleep(200 * time.Millisecond) - s.Require().Equal(1, s.getHistoryTasksCount()) - - s.archiveManager.StopHistoryArchiveTasksInterval(community.ID()) - s.archiveManager.historyArchiveTasksWaitGroup.Wait() - s.Require().Equal(0, s.getHistoryTasksCount()) -} - -func (s *ManagerSuite) TestStopHistoryArchiveTasksIntervals() { - err := s.archiveManager.StartTorrentClient() - s.Require().NoError(err) - defer s.archiveManager.Stop() //nolint: errcheck - - community, _, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - interval := 10 * time.Second - go s.archiveManager.StartHistoryArchiveTasksInterval(community, interval) - - time.Sleep(2 * time.Second) - - count := s.getHistoryTasksCount() - s.Require().Equal(count, 1) - - s.archiveManager.stopHistoryArchiveTasksIntervals() - - count = s.getHistoryTasksCount() - s.Require().Equal(count, 0) -} - -func (s *ManagerSuite) TestStopTorrentClient_ShouldStopHistoryArchiveTasks() { - err := s.archiveManager.StartTorrentClient() - s.Require().NoError(err) - defer s.archiveManager.Stop() //nolint: errcheck - - community, _, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - interval := 10 * time.Second - go s.archiveManager.StartHistoryArchiveTasksInterval(community, interval) - // Due to async exec we need to wait a bit until we check - // the task count. - time.Sleep(2 * time.Second) - - count := s.getHistoryTasksCount() - s.Require().Equal(count, 1) - - err = s.archiveManager.Stop() - s.Require().NoError(err) - - count = s.getHistoryTasksCount() - s.Require().Equal(count, 0) -} - -func (s *ManagerSuite) TestStartTorrentClient_DelayedUntilOnline() { - s.Require().False(s.archiveManager.torrentClientStarted()) - - s.archiveManager.SetOnline(true) - s.Require().True(s.archiveManager.torrentClientStarted()) -} - -func (s *ManagerSuite) TestCreateHistoryArchiveTorrent_WithoutMessages() { - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - // Time range of 7 days - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - // Partition of 7 days - partition := 7 * 24 * time.Hour - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromDB(community.ID(), topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - // There are no waku messages in the database so we don't expect - // any archives to be created - _, err = os.Stat(s.archiveManager.archiveDataFile(community.IDString())) - s.Require().Error(err) - _, err = os.Stat(s.archiveManager.archiveIndexFile(community.IDString())) - s.Require().Error(err) - _, err = os.Stat(torrentFile(s.archiveManager.torrentConfig.TorrentDir, community.IDString())) - s.Require().Error(err) -} - -func (s *ManagerSuite) TestCreateHistoryArchiveTorrent_ShouldCreateArchive() { - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - // Time range of 7 days - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - // Partition of 7 days, this should create a single archive - partition := 7 * 24 * time.Hour - - message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) - message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) - // This message is outside of the startDate-endDate range and should not - // be part of the archive - message3 := buildMessage(endDate.Add(2*time.Hour), topic, []byte{3}) - - err = s.manager.StoreWakuMessage(&message1) - s.Require().NoError(err) - err = s.manager.StoreWakuMessage(&message2) - s.Require().NoError(err) - err = s.manager.StoreWakuMessage(&message3) - s.Require().NoError(err) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromDB(community.ID(), topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - _, err = os.Stat(s.archiveManager.archiveDataFile(community.IDString())) - s.Require().NoError(err) - _, err = os.Stat(s.archiveManager.archiveIndexFile(community.IDString())) - s.Require().NoError(err) - _, err = os.Stat(torrentFile(s.archiveManager.torrentConfig.TorrentDir, community.IDString())) - s.Require().NoError(err) - - index, err := s.archiveManager.LoadHistoryArchiveIndexFromFile(s.manager.identity, community.ID()) - s.Require().NoError(err) - s.Require().Len(index.Archives, 1) - - totalData, err := os.ReadFile(s.archiveManager.archiveDataFile(community.IDString())) - s.Require().NoError(err) - - for _, metadata := range index.Archives { - archive := &protobuf.WakuMessageArchive{} - data := totalData[metadata.Offset : metadata.Offset+metadata.Size-metadata.Padding] - - err = proto.Unmarshal(data, archive) - s.Require().NoError(err) - - s.Require().Len(archive.Messages, 2) - } -} - -func (s *ManagerSuite) TestCreateHistoryArchiveTorrent_ShouldCreateMultipleArchives() { - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - // Time range of 3 weeks - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 21, 00, 00, 00, 0, time.UTC) - // 7 days partition, this should create three archives - partition := 7 * 24 * time.Hour - - message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) - message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) - // We expect 2 archives to be created for startDate - endDate of each - // 7 days of data. This message should end up in the second archive - message3 := buildMessage(startDate.Add(8*24*time.Hour), topic, []byte{3}) - // This one should end up in the third archive - message4 := buildMessage(startDate.Add(14*24*time.Hour), topic, []byte{4}) - - err = s.manager.StoreWakuMessage(&message1) - s.Require().NoError(err) - err = s.manager.StoreWakuMessage(&message2) - s.Require().NoError(err) - err = s.manager.StoreWakuMessage(&message3) - s.Require().NoError(err) - err = s.manager.StoreWakuMessage(&message4) - s.Require().NoError(err) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromDB(community.ID(), topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - index, err := s.archiveManager.LoadHistoryArchiveIndexFromFile(s.manager.identity, community.ID()) - s.Require().NoError(err) - s.Require().Len(index.Archives, 3) - - totalData, err := os.ReadFile(s.archiveManager.archiveDataFile(community.IDString())) - s.Require().NoError(err) - - // First archive has 2 messages - // Second archive has 1 message - // Third archive has 1 message - fromMap := map[uint64]int{ - uint64(startDate.Unix()): 2, - uint64(startDate.Add(partition).Unix()): 1, - uint64(startDate.Add(partition * 2).Unix()): 1, - } - - for _, metadata := range index.Archives { - archive := &protobuf.WakuMessageArchive{} - data := totalData[metadata.Offset : metadata.Offset+metadata.Size-metadata.Padding] - - err = proto.Unmarshal(data, archive) - s.Require().NoError(err) - s.Require().Len(archive.Messages, fromMap[metadata.Metadata.From]) - } -} - -func (s *ManagerSuite) TestCreateHistoryArchiveTorrent_ShouldAppendArchives() { - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - // Time range of 1 week - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - // 7 days partition, this should create one archive - partition := 7 * 24 * time.Hour - - message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) - err = s.manager.StoreWakuMessage(&message1) - s.Require().NoError(err) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromDB(community.ID(), topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - index, err := s.archiveManager.LoadHistoryArchiveIndexFromFile(s.manager.identity, community.ID()) - s.Require().NoError(err) - s.Require().Len(index.Archives, 1) - - // Time range of next week - startDate = time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - endDate = time.Date(2020, 1, 14, 00, 00, 00, 0, time.UTC) - - message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) - err = s.manager.StoreWakuMessage(&message2) - s.Require().NoError(err) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromDB(community.ID(), topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - index, err = s.archiveManager.LoadHistoryArchiveIndexFromFile(s.manager.identity, community.ID()) - s.Require().NoError(err) - s.Require().Len(index.Archives, 2) -} - -func (s *ManagerSuite) TestCreateHistoryArchiveTorrentFromMessages() { - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - // Time range of 7 days - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - // Partition of 7 days, this should create a single archive - partition := 7 * 24 * time.Hour - - message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) - message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) - // This message is outside of the startDate-endDate range and should not - // be part of the archive - message3 := buildMessage(endDate.Add(2*time.Hour), topic, []byte{3}) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromMessages(community.ID(), []*types.ReceivedMessage{&message1, &message2, &message3}, topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - _, err = os.Stat(s.archiveManager.archiveDataFile(community.IDString())) - s.Require().NoError(err) - _, err = os.Stat(s.archiveManager.archiveIndexFile(community.IDString())) - s.Require().NoError(err) - _, err = os.Stat(torrentFile(s.archiveManager.torrentConfig.TorrentDir, community.IDString())) - s.Require().NoError(err) - - index, err := s.archiveManager.LoadHistoryArchiveIndexFromFile(s.manager.identity, community.ID()) - s.Require().NoError(err) - s.Require().Len(index.Archives, 1) - - totalData, err := os.ReadFile(s.archiveManager.archiveDataFile(community.IDString())) - s.Require().NoError(err) - - for _, metadata := range index.Archives { - archive := &protobuf.WakuMessageArchive{} - data := totalData[metadata.Offset : metadata.Offset+metadata.Size-metadata.Padding] - - err = proto.Unmarshal(data, archive) - s.Require().NoError(err) - - s.Require().Len(archive.Messages, 2) - } -} - -func (s *ManagerSuite) TestCreateHistoryArchiveTorrentFromMessages_ShouldCreateMultipleArchives() { - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - // Time range of 3 weeks - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 21, 00, 00, 00, 0, time.UTC) - // 7 days partition, this should create three archives - partition := 7 * 24 * time.Hour - - message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) - message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) - // We expect 2 archives to be created for startDate - endDate of each - // 7 days of data. This message should end up in the second archive - message3 := buildMessage(startDate.Add(8*24*time.Hour), topic, []byte{3}) - // This one should end up in the third archive - message4 := buildMessage(startDate.Add(14*24*time.Hour), topic, []byte{4}) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromMessages(community.ID(), []*types.ReceivedMessage{&message1, &message2, &message3, &message4}, topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - index, err := s.archiveManager.LoadHistoryArchiveIndexFromFile(s.manager.identity, community.ID()) - s.Require().NoError(err) - s.Require().Len(index.Archives, 3) - - totalData, err := os.ReadFile(s.archiveManager.archiveDataFile(community.IDString())) - s.Require().NoError(err) - - // First archive has 2 messages - // Second archive has 1 message - // Third archive has 1 message - fromMap := map[uint64]int{ - uint64(startDate.Unix()): 2, - uint64(startDate.Add(partition).Unix()): 1, - uint64(startDate.Add(partition * 2).Unix()): 1, - } - - for _, metadata := range index.Archives { - archive := &protobuf.WakuMessageArchive{} - data := totalData[metadata.Offset : metadata.Offset+metadata.Size-metadata.Padding] - - err = proto.Unmarshal(data, archive) - s.Require().NoError(err) - s.Require().Len(archive.Messages, fromMap[metadata.Metadata.From]) - } -} - -func (s *ManagerSuite) TestCreateHistoryArchiveTorrentFromMessages_ShouldAppendArchives() { - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - // Time range of 1 week - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - // 7 days partition, this should create one archive - partition := 7 * 24 * time.Hour - - message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromMessages(community.ID(), []*types.ReceivedMessage{&message1}, topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - index, err := s.archiveManager.LoadHistoryArchiveIndexFromFile(s.manager.identity, community.ID()) - s.Require().NoError(err) - s.Require().Len(index.Archives, 1) - - // Time range of next week - startDate = time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - endDate = time.Date(2020, 1, 14, 00, 00, 00, 0, time.UTC) - - message2 := buildMessage(startDate.Add(2*time.Hour), topic, []byte{2}) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromMessages(community.ID(), []*types.ReceivedMessage{&message2}, topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - index, err = s.archiveManager.LoadHistoryArchiveIndexFromFile(s.manager.identity, community.ID()) - s.Require().NoError(err) - s.Require().Len(index.Archives, 2) -} - -func (s *ManagerSuite) TestSeedHistoryArchiveTorrent() { - err := s.archiveManager.StartTorrentClient() - s.Require().NoError(err) - defer s.archiveManager.Stop() //nolint: errcheck - - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - partition := 7 * 24 * time.Hour - - message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) - err = s.manager.StoreWakuMessage(&message1) - s.Require().NoError(err) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromDB(community.ID(), topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - err = s.archiveManager.SeedHistoryArchiveTorrent(community.ID()) - s.Require().NoError(err) - s.Require().Len(s.archiveManager.torrentTasks, 1) - - metaInfoHash := s.archiveManager.torrentTasks[community.IDString()] - torrent, ok := s.archiveManager.torrentClient.Torrent(metaInfoHash) - defer torrent.Drop() - - s.Require().Equal(ok, true) - s.Require().Equal(torrent.Seeding(), true) -} - -func (s *ManagerSuite) TestUnseedHistoryArchiveTorrent() { - err := s.archiveManager.StartTorrentClient() - s.Require().NoError(err) - defer s.archiveManager.Stop() //nolint: errcheck - - community, chatID, err := s.buildCommunityWithChat() - s.Require().NoError(err) - - topic := types.BytesToContentTopic(messaging.ToContentTopic(chatID)) - topics := []types.ContentTopic{topic} - - startDate := time.Date(2020, 1, 1, 00, 00, 00, 0, time.UTC) - endDate := time.Date(2020, 1, 7, 00, 00, 00, 0, time.UTC) - partition := 7 * 24 * time.Hour - - message1 := buildMessage(startDate.Add(1*time.Hour), topic, []byte{1}) - err = s.manager.StoreWakuMessage(&message1) - s.Require().NoError(err) - - _, err = s.archiveManager.CreateHistoryArchiveTorrentFromDB(community.ID(), topics, startDate, endDate, partition, false) - s.Require().NoError(err) - - err = s.archiveManager.SeedHistoryArchiveTorrent(community.ID()) - s.Require().NoError(err) - s.Require().Len(s.archiveManager.torrentTasks, 1) - - metaInfoHash := s.archiveManager.torrentTasks[community.IDString()] - - s.archiveManager.UnseedHistoryArchiveTorrent(community.ID()) - _, ok := s.archiveManager.torrentClient.Torrent(metaInfoHash) - s.Require().Equal(ok, false) -} - func (s *ManagerSuite) TestCheckChannelPermissions_NoPermissions() { m, _, tm := s.setupManagerForTokenPermissions() @@ -1661,15 +1162,6 @@ func (s *ManagerSuite) TestCheckAllChannelsPermissions() { s.Require().Len(response.Channels[chatID2].ViewOnlyPermissions.Permissions, 0) } -func buildTorrentConfig() *params.TorrentConfig { - return ¶ms.TorrentConfig{ - Enabled: true, - DataDir: os.TempDir() + "/archivedata", - TorrentDir: os.TempDir() + "/torrents", - Port: 0, - } -} - func buildMessage(timestamp time.Time, topic types.ContentTopic, hash []byte) types.ReceivedMessage { message := types.ReceivedMessage{ Sig: []byte{1}, diff --git a/protocol/communities/persistence.go b/protocol/communities/persistence.go index 7473b86ce63..2a2b67416a3 100644 --- a/protocol/communities/persistence.go +++ b/protocol/communities/persistence.go @@ -364,6 +364,19 @@ func (p *Persistence) GetByID(memberIdentity *ecdsa.PublicKey, id []byte) (*Comm return p.recordBundleToCommunity(r) } +// CommunityExists checks if a community with the given ID exists in the database. +// This is a lightweight alternative to GetByID when only existence checking is needed. +func (p *Persistence) CommunityExists(memberIdentity *ecdsa.PublicKey, id []byte) (bool, error) { + r, err := p.getByID(id, memberIdentity) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, err + } + return r != nil, nil +} + func (p *Persistence) SaveRequestToJoin(request *RequestToJoin) (err error) { tx, err := p.db.BeginTx(context.Background(), &sql.TxOptions{}) if err != nil { @@ -985,63 +998,83 @@ func (p *Persistence) HasCommunityArchiveInfo(communityID types3.HexBytes) (exis return exists, err } -func (p *Persistence) GetLastSeenMagnetlink(communityID types3.HexBytes) (string, error) { - var magnetlinkURI string - err := p.db.QueryRow(`SELECT last_magnetlink_uri FROM communities_archive_info WHERE community_id = ?`, communityID.String()).Scan(&magnetlinkURI) +func (p *Persistence) GetLastSeenArchiveLink(communityID types3.HexBytes) (string, error) { + var archiveLink string + err := p.db.QueryRow(`SELECT last_archive_link FROM communities_archive_info WHERE community_id = ?`, communityID.String()).Scan(&archiveLink) if err == sql.ErrNoRows { return "", nil } - return magnetlinkURI, err + return archiveLink, err } -func (p *Persistence) GetMagnetlinkMessageClock(communityID types3.HexBytes) (uint64, error) { - var magnetlinkClock uint64 - err := p.db.QueryRow(`SELECT magnetlink_clock FROM communities_archive_info WHERE community_id = ?`, communityID.String()).Scan(&magnetlinkClock) +func (p *Persistence) GetArchiveLinkMessageClock(communityID types3.HexBytes) (uint64, error) { + var archiveLinkClock uint64 + err := p.db.QueryRow(`SELECT archive_link_clock FROM communities_archive_info WHERE community_id = ?`, communityID.String()).Scan(&archiveLinkClock) if err == sql.ErrNoRows { return 0, nil } - return magnetlinkClock, err + return archiveLinkClock, err } -func (p *Persistence) SaveCommunityArchiveInfo(communityID types3.HexBytes, clock uint64, lastArchiveEndDate uint64) error { - _, err := p.db.Exec(`INSERT INTO communities_archive_info (magnetlink_clock, last_message_archive_end_date, community_id) VALUES (?, ?, ?)`, - clock, +func (p *Persistence) SaveCommunityArchiveInfo(communityID types3.HexBytes, archiveLinkClock uint64, lastArchiveEndDate uint64) error { + _, err := p.db.Exec(` + INSERT INTO communities_archive_info ( + community_id, archive_link_clock, last_message_archive_end_date + ) VALUES (?, ?, ?) + ON CONFLICT(community_id) DO UPDATE SET + archive_link_clock = excluded.archive_link_clock, + last_message_archive_end_date = excluded.last_message_archive_end_date`, + communityID.String(), + archiveLinkClock, lastArchiveEndDate, - communityID.String()) + ) return err } -func (p *Persistence) UpdateMagnetlinkMessageClock(communityID types3.HexBytes, clock uint64) error { - _, err := p.db.Exec(`UPDATE communities_archive_info SET - magnetlink_clock = ? - WHERE community_id = ?`, - clock, - communityID.String()) +func (p *Persistence) UpdateArchiveLinkMessageClock(communityID types3.HexBytes, archiveLinkClock uint64) error { + _, err := p.db.Exec(` + INSERT INTO communities_archive_info (community_id, archive_link_clock) + VALUES (?, ?) + ON CONFLICT(community_id) DO UPDATE SET + archive_link_clock = excluded.archive_link_clock`, + communityID.String(), + archiveLinkClock, + ) return err } -func (p *Persistence) UpdateLastSeenMagnetlink(communityID types3.HexBytes, magnetlinkURI string) error { - _, err := p.db.Exec(`UPDATE communities_archive_info SET - last_magnetlink_uri = ? - WHERE community_id = ?`, - magnetlinkURI, - communityID.String()) +func (p *Persistence) UpdateLastSeenArchiveLink(communityID types3.HexBytes, archiveLink string) error { + _, err := p.db.Exec(` + INSERT INTO communities_archive_info (community_id, last_archive_link) + VALUES (?, ?) + ON CONFLICT(community_id) DO UPDATE SET + last_archive_link = excluded.last_archive_link`, + communityID.String(), + archiveLink, + ) return err } func (p *Persistence) SaveLastMessageArchiveEndDate(communityID types3.HexBytes, endDate uint64) error { - _, err := p.db.Exec(`INSERT INTO communities_archive_info (last_message_archive_end_date, community_id) VALUES (?, ?)`, + _, err := p.db.Exec(` + INSERT INTO communities_archive_info (community_id, last_message_archive_end_date) VALUES (?, ?) + ON CONFLICT(community_id) DO UPDATE SET + last_message_archive_end_date = excluded.last_message_archive_end_date`, + communityID.String(), endDate, - communityID.String()) + ) return err } func (p *Persistence) UpdateLastMessageArchiveEndDate(communityID types3.HexBytes, endDate uint64) error { - _, err := p.db.Exec(`UPDATE communities_archive_info SET - last_message_archive_end_date = ? - WHERE community_id = ?`, + _, err := p.db.Exec(` + INSERT INTO communities_archive_info (community_id, last_message_archive_end_date) + VALUES (?, ?) + ON CONFLICT(community_id) DO UPDATE SET + last_message_archive_end_date = excluded.last_message_archive_end_date`, + communityID.String(), endDate, - communityID.String()) + ) return err } diff --git a/protocol/communities_messenger_token_permissions_test.go b/protocol/communities_messenger_token_permissions_test.go index 75745b35c15..8f227198056 100644 --- a/protocol/communities_messenger_token_permissions_test.go +++ b/protocol/communities_messenger_token_permissions_test.go @@ -29,6 +29,8 @@ import ( messagingtypes "github.com/status-im/status-go/pkg/messaging/types" "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" + "github.com/status-im/status-go/protocol/communities/archive" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/requests" "github.com/status-im/status-go/services/wallet/thirdparty" @@ -2208,12 +2210,24 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestImportDecryptedArchiveMe } // Share archive directory between all users - s.owner.archiveManager.SetTorrentConfig(&torrentConfig) - s.bob.archiveManager.SetTorrentConfig(&torrentConfig) + amc := &archivetypes.ArchiveManagerConfig{ + TorrentConfig: &torrentConfig, + } + + s.owner.SetupArchiveManager(amc) + s.bob.SetupArchiveManager(amc) + s.owner.config.messengerSignalsHandler = &MessengerSignalsHandlerMock{} s.bob.config.messengerSignalsHandler = &MessengerSignalsHandlerMock{} - archiveIDs, err := s.owner.archiveManager.CreateHistoryArchiveTorrentFromDB(community.ID(), topics, startDate, endDate, partition, community.Encrypted()) + archiveManager, ok := s.owner.archiveManager.(*archive.ArchiveManager) + s.Require().True(ok) + + torrentBackend, err := archiveManager.GetTorrentBackend() + s.Require().NoError(err) + s.Require().NotNil(torrentBackend) + + archiveIDs, err := torrentBackend.CreateHistoryArchiveFromDB(community.ID(), topics, startDate, endDate, partition, community.Encrypted()) s.Require().NoError(err) s.Require().Len(archiveIDs, 1) @@ -2245,12 +2259,27 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestImportDecryptedArchiveMe // https://github.com/status-im/status-go/blob/6c82a6c2be7ebed93bcae3b9cf5053da3820de50/protocol/communities/manager.go#L4403 // Ensure owner has archive - archiveIndex, err := s.owner.archiveManager.LoadHistoryArchiveIndexFromFile(s.owner.identity, community.ID()) + + archiveManager, ok = s.owner.archiveManager.(*archive.ArchiveManager) + s.Require().True(ok) + + torrentBackend, err = archiveManager.GetTorrentBackend() + s.Require().NoError(err) + s.Require().NotNil(torrentBackend) + + archiveIndex, err := torrentBackend.LoadHistoryArchiveIndexFromFile(s.owner.identity, community.ID()) s.Require().NoError(err) s.Require().Len(archiveIndex.Archives, 1) // Ensure bob has archive (because they share same local directory) - archiveIndex, err = s.bob.archiveManager.LoadHistoryArchiveIndexFromFile(s.bob.identity, community.ID()) + archiveManager, ok = s.bob.archiveManager.(*archive.ArchiveManager) + s.Require().True(ok) + + torrentBackend, err = archiveManager.GetTorrentBackend() + s.Require().NoError(err) + s.Require().NotNil(torrentBackend) + + archiveIndex, err = torrentBackend.LoadHistoryArchiveIndexFromFile(s.bob.identity, community.ID()) s.Require().NoError(err) s.Require().Len(archiveIndex.Archives, 1) @@ -2266,7 +2295,7 @@ func (s *MessengerCommunitiesTokenPermissionsSuite) TestImportDecryptedArchiveMe close(s.bob.importDelayer.wait) }) cancel := make(chan struct{}) - err = s.bob.importHistoryArchives(community.ID(), cancel) + err = s.bob.importHistoryArchives(community.ID(), cancel, "") s.Require().NoError(err) // Ensure message1 wasn't imported, as it's encrypted, and we don't have access to the channel diff --git a/protocol/messenger.go b/protocol/messenger.go index 7c85a018c37..0fb546bc1b7 100644 --- a/protocol/messenger.go +++ b/protocol/messenger.go @@ -45,6 +45,8 @@ import ( "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" + "github.com/status-im/status-go/protocol/communities/archive" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" "github.com/status-im/status-go/protocol/ens" "github.com/status-im/status-go/protocol/identity/alias" "github.com/status-im/status-go/protocol/identity/identicon" @@ -104,7 +106,7 @@ type Messenger struct { pushNotificationClient *pushnotificationclient.Client pushNotificationServer PushNotificationServer communitiesManager *communities.Manager - archiveManager communities.ArchiveService + archiveManager archive.ArchiveService communitiesKeyDistributor communities.KeyDistributor accountsManager AccountsManager mentionsManager *MentionManager @@ -369,7 +371,7 @@ func NewMessenger( return nil, err } - amc := &communities.ArchiveManagerConfig{ + amc := &archivetypes.ArchiveManagerConfig{ TorrentConfig: c.torrentConfig, Logger: logger, Persistence: communitiesManager.GetPersistence(), @@ -381,7 +383,7 @@ func NewMessenger( // Depending on the OS go will choose whether to use the "communities/manager_archive_nop.go" or // "communities/manager_archive.go" version of this function based on the build instructions for those files. // See those file for more details. - archiveManager := communities.NewArchiveManager(amc) + archiveManager := archive.NewArchiveManager(amc) settings, err := accounts.NewDB(database) if err != nil { @@ -443,7 +445,12 @@ func NewMessenger( shutdownTasks: []func() error{ pushNotificationClient.Stop, communitiesManager.Stop, - archiveManager.Stop, + func() error { + if messenger.archiveManager == nil { + return nil + } + return messenger.archiveManager.Stop() + }, }, logger: logger, tracer: c.tracer, @@ -493,6 +500,26 @@ func NewMessenger( return messenger, nil } +func (m *Messenger) SetupArchiveManager(amc *archivetypes.ArchiveManagerConfig) { + if amc.Logger == nil { + amc.Logger = m.logger + } + if amc.Persistence == nil { + amc.Persistence = m.communitiesManager.GetPersistence() + } + if amc.Messaging == nil { + amc.Messaging = m.messaging + } + if amc.Identity == nil { + amc.Identity = m.identity + } + if amc.Publisher == nil { + amc.Publisher = m.communitiesManager + } + + m.archiveManager = archive.NewArchiveManager(amc) +} + func (m *Messenger) processSentMessage(id string) error { rawMessage, err := m.persistence.RawMessageByID(id) // If we have no raw message, we create a temporary one, so that @@ -554,9 +581,10 @@ func (m *Messenger) SetPaused(paused bool) { m.messaging.ResumeTransport() } } - if m.archiveManager != nil { - m.archiveManager.SetPaused(paused) - } + // ToDo: the current ArchiveManager does not provide SetPaused method yet + // if m.archiveManager != nil { + // m.archiveManager.SetPaused(paused) + // } } func (m *Messenger) isPaused() bool { @@ -651,7 +679,7 @@ func (m *Messenger) Start() (*MessengerResponse, error) { return nil, err } - if m.archiveManager.IsReady() { + if m.archiveManager.IsStarted() { m.shutdownWaitGroup.Add(1) go func() { defer gocommon.LogOnPanic() @@ -782,9 +810,7 @@ func (m *Messenger) handleConnectionChange(online bool) { } // Update torrent manager - if m.archiveManager != nil { - m.archiveManager.SetOnline(online) - } + m.archiveManager.SetOnline(online) // Publish contact code if online && m.shouldPublishContactCode { diff --git a/protocol/messenger_communities.go b/protocol/messenger_communities.go index 615df9a6611..ab9033d8d96 100644 --- a/protocol/messenger_communities.go +++ b/protocol/messenger_communities.go @@ -32,6 +32,7 @@ import ( types2 "github.com/status-im/status-go/pkg/messaging/types" "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" "github.com/status-im/status-go/protocol/communities/token" "github.com/status-im/status-go/protocol/contacts" "github.com/status-im/status-go/protocol/discord" @@ -257,7 +258,9 @@ func (m *Messenger) handleCommunitiesHistoryArchivesSubscription(c chan *communi if sub.HistoryArchivesSeedingSignal != nil { - m.config.messengerSignalsHandler.HistoryArchivesSeeding(sub.HistoryArchivesSeedingSignal.CommunityID) + m.config.messengerSignalsHandler.HistoryArchivesSeeding( + sub.HistoryArchivesSeedingSignal.CommunityID, + ) c, err := m.communitiesManager.GetByIDString(sub.HistoryArchivesSeedingSignal.CommunityID) if err != nil { @@ -265,9 +268,9 @@ func (m *Messenger) handleCommunitiesHistoryArchivesSubscription(c chan *communi } if c.IsControlNode() { - err := m.dispatchMagnetlinkMessage(sub.HistoryArchivesSeedingSignal.CommunityID) + err := m.dispatchArchiveLinkMessage(sub.HistoryArchivesSeedingSignal.CommunityID) if err != nil { - m.logger.Debug("failed to dispatch magnetlink message", zap.Error(err)) + m.logger.Debug("failed to dispatch archiveLink message", zap.Error(err)) } } } @@ -1987,14 +1990,17 @@ func (m *Messenger) acceptRequestToJoinCommunity(requestToJoin *communities.Requ CommunityDescriptionProtocolMessage: descriptionMessage, } - // The purpose of this torrent code is to get the 'magnetlink' to populate 'requestToJoinResponseProto.MagnetUri' - if m.archiveManager.IsReady() && m.archiveManager.TorrentFileExists(community.IDString()) { - magnetlink, err := m.archiveManager.GetHistoryArchiveMagnetlink(community.ID()) + if m.archiveManager.IsStarted() { + m.logger.Debug("[Messenger][acceptRequestToJoinCommunity] checking if currently seeding", zap.String("communityID", community.IDString())) + archiveLink, err := m.communitiesManager.GetLastSeenArchiveLink(community.ID()) if err != nil { - m.logger.Warn("couldn't get magnet link for community", zap.Error(err)) + m.logger.Warn("couldn't get archive link for community", zap.Error(err)) return nil, err } - requestToJoinResponseProto.MagnetUri = magnetlink + if m.archiveManager.IsSeedingHistoryArchive(community.ID(), archiveLink) { + m.logger.Debug("[Messenger][acceptRequestToJoinCommunity] setting requestToJoinResponseProto.ArchiveLink", zap.String("communityID", community.IDString()), zap.String("archiveLink", archiveLink)) + requestToJoinResponseProto.ArchiveLink = archiveLink + } } payload, err := proto.Marshal(requestToJoinResponseProto) @@ -2530,8 +2536,8 @@ func (m *Messenger) CreateCommunity(request *requests.CreateCommunity, createDef return nil, err } - if m.config.torrentConfig != nil && m.config.torrentConfig.Enabled && communitySettings.HistoryArchiveSupportEnabled { - go m.archiveManager.StartHistoryArchiveTasksInterval(community, messageArchiveInterval) + if m.archiveManager.IsStarted() && communitySettings.HistoryArchiveSupportEnabled { + go m.archiveManager.StartHistoryArchiveTasksInterval(community.ID(), community.UniversalChatID(), community.Encrypted(), messageArchiveInterval) } return response, nil @@ -2709,10 +2715,14 @@ func (m *Messenger) EditCommunity(request *requests.EditCommunity) (*MessengerRe id := community.ID() - if m.archiveManager.IsReady() { + if m.archiveManager.IsStarted() { + lastSeenArchiveLink, err := m.communitiesManager.GetLastSeenArchiveLink(id) + if err != nil { + return nil, err + } if !communitySettings.HistoryArchiveSupportEnabled { m.archiveManager.StopHistoryArchiveTasksInterval(id) - } else if !m.archiveManager.IsSeedingHistoryArchiveTorrent(id) { + } else if !m.archiveManager.IsSeedingHistoryArchive(id, lastSeenArchiveLink) { var communities []*communities.Community communities = append(communities, community) go m.InitHistoryArchiveTasks(communities) @@ -2789,7 +2799,7 @@ func (m *Messenger) ImportCommunity(ctx context.Context, key *ecdsa.PrivateKey) return nil, err } - if m.archiveManager.IsReady() { + if m.archiveManager.IsStarted() { var communities []*communities.Community communities = append(communities, community) go m.InitHistoryArchiveTasks(communities) @@ -3572,12 +3582,14 @@ func (m *Messenger) InitHistoryArchiveTasks(communities []*communities.Community continue } - // Check if there's already a torrent file for this community and seed it - if m.archiveManager.TorrentFileExists(c.IDString()) { - err = m.archiveManager.SeedHistoryArchiveTorrent(c.ID()) + lastSeenArchiveLink, err := m.communitiesManager.GetLastSeenArchiveLink(c.ID()) + if err == nil { + err = m.archiveManager.SeedHistoryArchive(c.ID(), lastSeenArchiveLink) if err != nil { - m.logger.Error("failed to seed history archive", zap.Error(err)) + m.logger.Error("[LogosStorage][init_history_archive_tasks] failed to seed history archive", zap.Error(err)) } + } else { + m.logger.Error("[LogosStorage][init_history_archive_tasks] failed to get last seen archive link", zap.Error(err)) } filters, err := m.archiveManager.GetCommunityChatsFilters(c.ID()) @@ -3586,9 +3598,14 @@ func (m *Messenger) InitHistoryArchiveTasks(communities []*communities.Community continue } + filter := m.messaging.ChatFilterByChatID(c.UniversalChatID()) + if filter != nil { + filters = append(filters, filter) + } + if len(filters) == 0 { m.logger.Debug("no filters or chats for this community starting interval", zap.String("id", c.IDString())) - go m.archiveManager.StartHistoryArchiveTasksInterval(c, messageArchiveInterval) + go m.archiveManager.StartHistoryArchiveTasksInterval(c.ID(), c.UniversalChatID(), c.Encrypted(), messageArchiveInterval) continue } @@ -3598,11 +3615,6 @@ func (m *Messenger) InitHistoryArchiveTasks(communities []*communities.Community topics = append(topics, filter.ContentTopic()) } - filter := m.messaging.ChatFilterByChatID(c.UniversalChatID()) - if filter != nil { - filters = append(filters, filter) - } - // First we need to know the timestamp of the latest waku message // we've received for this community, so we can request messages we've // possibly missed since then @@ -3645,14 +3657,19 @@ func (m *Messenger) InitHistoryArchiveTasks(communities []*communities.Community if lastArchiveEndDateTimestamp == 0 { // No prior messages to be archived, so we just kick off the archive creation loop // for future archives - go m.archiveManager.StartHistoryArchiveTasksInterval(c, messageArchiveInterval) + go m.archiveManager.StartHistoryArchiveTasksInterval(c.ID(), c.UniversalChatID(), c.Encrypted(), messageArchiveInterval) } else if durationSinceLastArchive < messageArchiveInterval { // Last archive is less than `interval` old, wait until `interval` is complete, // then create archive and kick off archive creation loop for future archives // Seed current archive in the meantime - err := m.archiveManager.SeedHistoryArchiveTorrent(c.ID()) - if err != nil { - m.logger.Error("failed to seed history archive", zap.Error(err)) + lastSeenArchiveLink, err := m.communitiesManager.GetLastSeenArchiveLink(c.ID()) + if err == nil { + err := m.archiveManager.SeedHistoryArchive(c.ID(), lastSeenArchiveLink) + if err != nil { + m.logger.Error("[LogosStorage][init_history_archive_tasks] failed to seed history archive", zap.Error(err)) + } + } else { + m.logger.Error("[LogosStorage][init_history_archive_tasks] failed to get last seen archive link", zap.Error(err)) } timeToNextInterval := messageArchiveInterval - durationSinceLastArchive @@ -3662,7 +3679,7 @@ func (m *Messenger) InitHistoryArchiveTasks(communities []*communities.Community if err != nil { m.logger.Error("failed to get create and seed history archive", zap.Error(err)) } - go m.archiveManager.StartHistoryArchiveTasksInterval(c, messageArchiveInterval) + go m.archiveManager.StartHistoryArchiveTasksInterval(c.ID(), c.UniversalChatID(), c.Encrypted(), messageArchiveInterval) }) } else { // Looks like the last archive was generated more than `interval` @@ -3673,7 +3690,7 @@ func (m *Messenger) InitHistoryArchiveTasks(communities []*communities.Community m.logger.Error("failed to get create and seed history archive", zap.Error(err)) } - go m.archiveManager.StartHistoryArchiveTasksInterval(c, messageArchiveInterval) + go m.archiveManager.StartHistoryArchiveTasksInterval(c.ID(), c.UniversalChatID(), c.Encrypted(), messageArchiveInterval) } } } @@ -3726,7 +3743,7 @@ func (m *Messenger) resumeHistoryArchivesImport(communityID types3.HexBytes) err } // Create new task - task := &communities.HistoryArchiveDownloadTask{ + task := &archivetypes.HistoryArchiveDownloadTask{ CancelChan: make(chan struct{}), Waiter: *new(sync.WaitGroup), Cancelled: false, @@ -3740,7 +3757,13 @@ func (m *Messenger) resumeHistoryArchivesImport(communityID types3.HexBytes) err go func() { defer gocommon.LogOnPanic() defer task.Waiter.Done() - err := m.importHistoryArchives(communityID, task.CancelChan) + defer m.archiveManager.RemoveHistoryArchiveDownloadTask(communityID.String()) + lastSeenArchiveLink, err := m.communitiesManager.GetLastSeenArchiveLink(communityID) + if err != nil { + m.logger.Error("failed to get last seen archive link", zap.Error(err)) + return + } + err = m.importHistoryArchives(communityID, task.CancelChan, lastSeenArchiveLink) if err != nil { m.logger.Error("failed to import history archives", zap.Error(err)) } @@ -3757,7 +3780,7 @@ func (m *Messenger) SlowdownArchivesImport() { m.importRateLimiter.SetLimit(rate.Every(importSlowRate)) } -func (m *Messenger) importHistoryArchives(communityID types3.HexBytes, cancel chan struct{}) error { +func (m *Messenger) importHistoryArchives(communityID types3.HexBytes, cancel chan struct{}, archiveLink string) error { importTicker := time.NewTicker(100 * time.Millisecond) defer importTicker.Stop() @@ -3815,7 +3838,8 @@ importMessageArchivesLoop: // wait for all archives to be processed first downloadedArchiveID := archiveIDsToImport[0] - archiveMessages, err := m.archiveManager.ExtractMessagesFromHistoryArchive(communityID, downloadedArchiveID) + archiveMessages, err := m.archiveManager.LoadArchiveMessages(ctx, communityID, archiveLink, downloadedArchiveID) + if err != nil { if errors.Is(err, types2.ErrHashRatchetGroupIDNotFound) { // In case we're missing hash ratchet keys, best we can do is @@ -3861,24 +3885,24 @@ importMessageArchivesLoop: return nil } -func (m *Messenger) dispatchMagnetlinkMessage(communityID string) error { +func (m *Messenger) dispatchArchiveLinkMessage(communityID string) error { community, err := m.communitiesManager.GetByIDString(communityID) if err != nil { return err } - magnetlink, err := m.archiveManager.GetHistoryArchiveMagnetlink(community.ID()) + archiveLink, err := m.archiveManager.GetHistoryArchiveLink(community.ID()) if err != nil { return err } - magnetLinkMessage := &protobuf.CommunityMessageArchiveMagnetlink{ - Clock: m.getTimesource().GetCurrentTime(), - MagnetUri: magnetlink, + archiveLinkMessage := &protobuf.CommunityMessageArchiveLink{ + Clock: m.getTimesource().GetCurrentTime(), + ArchiveLink: archiveLink, } - encodedMessage, err := proto.Marshal(magnetLinkMessage) + encodedMessage, err := proto.Marshal(archiveLinkMessage) if err != nil { return err } @@ -3888,7 +3912,7 @@ func (m *Messenger) dispatchMagnetlinkMessage(communityID string) error { LocalChatID: chatID, Sender: community.PrivateKey(), Payload: encodedMessage, - MessageType: protobuf.ApplicationMetadataMessage_COMMUNITY_MESSAGE_ARCHIVE_MAGNETLINK, + MessageType: protobuf.ApplicationMetadataMessage_COMMUNITY_MESSAGE_ARCHIVE_LINK, SkipGroupMessageWrap: true, PubsubTopic: community.PubsubTopic(), Priority: &types2.LowPriority, @@ -3899,11 +3923,11 @@ func (m *Messenger) dispatchMagnetlinkMessage(communityID string) error { return err } - err = m.communitiesManager.UpdateCommunityDescriptionMagnetlinkMessageClock(community.ID(), magnetLinkMessage.Clock) + err = m.communitiesManager.UpdateCommunityDescriptionArchiveLinkMessageClock(community.ID(), archiveLinkMessage.Clock) if err != nil { return err } - return m.communitiesManager.UpdateMagnetlinkMessageClock(community.ID(), magnetLinkMessage.Clock) + return m.communitiesManager.UpdateArchiveLinkMessageClock(community.ID(), archiveLinkMessage.Clock) } func (m *Messenger) EnableCommunityHistoryArchiveProtocol() error { @@ -3923,8 +3947,14 @@ func (m *Messenger) EnableCommunityHistoryArchiveProtocol() error { } m.config.torrentConfig = &nodeConfig.TorrentConfig - m.archiveManager.SetTorrentConfig(&nodeConfig.TorrentConfig) - err = m.archiveManager.StartTorrentClient() + + amc := &archivetypes.ArchiveManagerConfig{ + TorrentConfig: &nodeConfig.TorrentConfig, + } + + m.SetupArchiveManager(amc) + + err = m.archiveManager.Start() if err != nil { return err } @@ -3960,11 +3990,18 @@ func (m *Messenger) DisableCommunityHistoryArchiveProtocol() error { nodeConfig.TorrentConfig.Enabled = false err = m.settings.SaveSetting("node-config", nodeConfig) - m.config.torrentConfig = &nodeConfig.TorrentConfig - m.archiveManager.SetTorrentConfig(&nodeConfig.TorrentConfig) if err != nil { return err } + + m.config.torrentConfig = &nodeConfig.TorrentConfig + + amc := &archivetypes.ArchiveManagerConfig{ + TorrentConfig: &nodeConfig.TorrentConfig, + } + + m.SetupArchiveManager(amc) + if m.config.messengerSignalsHandler != nil { m.config.messengerSignalsHandler.HistoryArchivesProtocolDisabled() } diff --git a/protocol/messenger_communities_import_discord.go b/protocol/messenger_communities_import_discord.go index 9e0591a3940..ad40afe3e82 100644 --- a/protocol/messenger_communities_import_discord.go +++ b/protocol/messenger_communities_import_discord.go @@ -960,7 +960,7 @@ func (m *Messenger) RequestImportDiscordChannel(request *requests.ImportDiscordC startDate := time.Unix(int64(exportData.OldestMessageTimestamp), 0) endDate := time.Now() - _, err = m.archiveManager.CreateHistoryArchiveTorrentFromMessages( + _, err = m.archiveManager.CreateHistoryArchiveFromMessages( request.CommunityID, wakuMessages, topics, @@ -978,13 +978,18 @@ func (m *Messenger) RequestImportDiscordChannel(request *requests.ImportDiscordC m.logger.Error("Failed to get community settings", zap.Error(err)) continue } - if m.archiveManager.IsReady() && communitySettings.HistoryArchiveSupportEnabled { - - err = m.archiveManager.SeedHistoryArchiveTorrent(request.CommunityID) - if err != nil { - m.logger.Error("failed to seed history archive", zap.Error(err)) + if m.archiveManager.IsStarted() && communitySettings.HistoryArchiveSupportEnabled { + lastSeenArchiveLink, err := m.communitiesManager.GetLastSeenArchiveLink(request.CommunityID) + if err == nil { + err = m.archiveManager.SeedHistoryArchive(request.CommunityID, lastSeenArchiveLink) + if err != nil { + m.logger.Error("[LogosStorage][request_import_discord_channel] failed to seed history archive link", zap.Error(err), zap.String("lastSeenArchiveLink", lastSeenArchiveLink)) + } + } else { + m.logger.Error("[LogosStorage][request_import_discord_channel] failed to get last seen archive link", zap.Error(err)) } - go m.archiveManager.StartHistoryArchiveTasksInterval(community, messageArchiveInterval) + m.logger.Debug("[LogosStorage][request_import_discord_channel] starting history archive tasks interval") + go m.archiveManager.StartHistoryArchiveTasksInterval(community.ID(), community.UniversalChatID(), community.Encrypted(), messageArchiveInterval) } } @@ -1732,7 +1737,7 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor startDate := time.Unix(int64(exportData.OldestMessageTimestamp), 0) endDate := time.Now() - _, err = m.archiveManager.CreateHistoryArchiveTorrentFromMessages( + _, err = m.archiveManager.CreateHistoryArchiveFromMessages( discordCommunity.ID(), wakuMessages, topics, @@ -1746,13 +1751,18 @@ func (m *Messenger) RequestImportDiscordCommunity(request *requests.ImportDiscor continue } - if m.archiveManager.IsReady() && communitySettings.HistoryArchiveSupportEnabled { - - err = m.archiveManager.SeedHistoryArchiveTorrent(discordCommunity.ID()) - if err != nil { - m.logger.Error("failed to seed history archive", zap.Error(err)) + if m.archiveManager.IsStarted() && communitySettings.HistoryArchiveSupportEnabled { + lastSeenArchiveLink, err := m.communitiesManager.GetLastSeenArchiveLink(discordCommunity.ID()) + if err == nil { + err = m.archiveManager.SeedHistoryArchive(discordCommunity.ID(), lastSeenArchiveLink) + if err != nil { + m.logger.Error("[LogosStorage][RequestImportDiscordCommunity] failed to seed history archive", zap.Error(err), zap.String("lastSeenArchiveLink", lastSeenArchiveLink)) + } + } else { + m.logger.Error("[LogosStorage][RequestImportDiscordCommunity] failed to get last seen archive link", zap.Error(err)) } - go m.archiveManager.StartHistoryArchiveTasksInterval(discordCommunity, messageArchiveInterval) + m.logger.Debug("[LogosStorage][TORRENT][RequestImportDiscordCommunity] starting history archive tasks interval") + go m.archiveManager.StartHistoryArchiveTasksInterval(discordCommunity.ID(), discordCommunity.UniversalChatID(), discordCommunity.Encrypted(), messageArchiveInterval) } } diff --git a/protocol/messenger_handler.go b/protocol/messenger_handler.go index a980b951e47..0dfbbb4e890 100644 --- a/protocol/messenger_handler.go +++ b/protocol/messenger_handler.go @@ -34,6 +34,8 @@ import ( "github.com/status-im/status-go/internal/images" "github.com/status-im/status-go/protocol/common" "github.com/status-im/status-go/protocol/communities" + archivecommons "github.com/status-im/status-go/protocol/communities/archive/commons" + archivetypes "github.com/status-im/status-go/protocol/communities/archive/types" "github.com/status-im/status-go/protocol/protobuf" "github.com/status-im/status-go/protocol/requests" "github.com/status-im/status-go/protocol/syncing" @@ -1231,7 +1233,8 @@ func (m *Messenger) HandleSyncPairInstallation(ctx context.Context, state *Recei return nil } -func (m *Messenger) HandleHistoryArchiveMagnetlinkMessage(state *ReceivedMessageState, communityPubKey *ecdsa.PublicKey, magnetlink string, clock uint64) error { +func (m *Messenger) HandleHistoryArchiveLinkMessage(state *ReceivedMessageState, communityPubKey *ecdsa.PublicKey, archiveLink string, clock uint64) error { + m.logger.Debug("[LogosStorage][HandleHistoryArchiveLinkMessage] Handling history archive link message", zap.String("archiveLink", archiveLink)) id := types3.HexBytes(crypto.CompressPubkey(communityPubKey)) community, err := m.communitiesManager.GetByID(id) @@ -1252,28 +1255,36 @@ func (m *Messenger) HandleHistoryArchiveMagnetlinkMessage(state *ReceivedMessage return nil } - if m.archiveManager.IsReady() && settings.HistoryArchiveSupportEnabled { - lastClock, err := m.communitiesManager.GetMagnetlinkMessageClock(id) + if m.archiveManager.IsStarted() && settings.HistoryArchiveSupportEnabled { + m.logger.Debug("[LogosStorage][HandleHistoryArchiveLinkMessage] ArchiveManager is started and history archive support is enabled", zap.String("communityID", community.IDString())) + + lastArchiveLinkClock, err := m.communitiesManager.GetArchiveLinkMessageClock(id) if err != nil { return err } - lastSeenMagnetlink, err := m.communitiesManager.GetLastSeenMagnetlink(id) + + lastSeenArchiveLink, err := m.communitiesManager.GetLastSeenArchiveLink(id) if err != nil { return err } - // We are only interested in a community archive magnet link + // We are only interested in a community archive archiveLink // if it originates from a community that the current account is // part of and doesn't own the private key at the same time - if !community.IsControlNode() && community.Joined() && clock >= lastClock { - if lastSeenMagnetlink == magnetlink { - m.logger.Debug("already processed this magnetlink") + m.logger.Debug("[LogosStorage][HandleHistoryArchiveLinkMessage] Checking community membership and lastArchiveLinkClock", zap.String("communityID", community.IDString()), zap.Uint64("lastArchiveLinkClock", lastArchiveLinkClock), zap.Uint64("messageClock", clock)) + + if !community.IsControlNode() && community.Joined() && clock >= lastArchiveLinkClock { + if lastSeenArchiveLink == archiveLink { + m.logger.Debug("already processed this archiveLink") return nil } - m.archiveManager.UnseedHistoryArchiveTorrent(id) + // All checks passed - proceed with download + m.logger.Debug("[LogosStorage][HandleHistoryArchiveLinkMessage] Unseeding existing history archive link for community (if any)", zap.String("communityID", community.IDString())) + + m.archiveManager.UnseedHistoryArchive(id, lastSeenArchiveLink) currentTask := m.archiveManager.GetHistoryArchiveDownloadTask(id.String()) - go func(currentTask *communities.HistoryArchiveDownloadTask, communityID types3.HexBytes) { + go func(currentTask *archivetypes.HistoryArchiveDownloadTask, communityID types3.HexBytes) { defer gocommon.LogOnPanic() // Cancel ongoing download/import task if currentTask != nil && !currentTask.IsCancelled() { @@ -1282,7 +1293,7 @@ func (m *Messenger) HandleHistoryArchiveMagnetlinkMessage(state *ReceivedMessage } // Create new task - task := &communities.HistoryArchiveDownloadTask{ + task := &archivetypes.HistoryArchiveDownloadTask{ CancelChan: make(chan struct{}), Waiter: *new(sync.WaitGroup), Cancelled: false, @@ -1293,26 +1304,31 @@ func (m *Messenger) HandleHistoryArchiveMagnetlinkMessage(state *ReceivedMessage // this wait groups tracks the ongoing task for a particular community task.Waiter.Add(1) defer task.Waiter.Done() + defer m.archiveManager.RemoveHistoryArchiveDownloadTask(communityID.String()) // this wait groups tracks all ongoing tasks across communities m.shutdownWaitGroup.Add(1) defer m.shutdownWaitGroup.Done() - m.downloadAndImportHistoryArchives(communityID, magnetlink, task.CancelChan) + + m.logger.Debug("[LogosStorage][HandleHistoryArchiveLinkMessage] Calling downloadAndImportHistoryArchives", zap.String("archiveLink", archiveLink)) + + m.downloadAndImportHistoryArchives(communityID, archiveLink, task.CancelChan) }(currentTask, id) - return m.communitiesManager.UpdateMagnetlinkMessageClock(id, clock) + m.logger.Debug("[LogosStorage][HandleHistoryArchiveLinkMessage] Updating archive link message clock", zap.String("communityID", community.IDString()), zap.Uint64("clock", clock)) + return m.communitiesManager.UpdateArchiveLinkMessageClock(id, clock) } } return nil } -func (m *Messenger) downloadAndImportHistoryArchives(id types3.HexBytes, magnetlink string, cancel chan struct{}) { - downloadTaskInfo, err := m.archiveManager.DownloadHistoryArchivesByMagnetlink(id, magnetlink, cancel) +func (m *Messenger) downloadAndImportHistoryArchives(id types3.HexBytes, archiveLink string, cancel chan struct{}) { + downloadTaskInfo, err := m.archiveManager.DownloadHistoryArchives(id, archiveLink, cancel) if err != nil { logMsg := "failed to download history archive data" - if err == communities.ErrTorrentTimedout { - m.logger.Debug("torrent has timed out, trying once more...") - downloadTaskInfo, err = m.archiveManager.DownloadHistoryArchivesByMagnetlink(id, magnetlink, cancel) + if err == archivecommons.ErrArchiveTimedout { + m.logger.Debug("archive has timed out, trying once more...") + downloadTaskInfo, err = m.archiveManager.DownloadHistoryArchives(id, archiveLink, cancel) if err != nil { m.logger.Error(logMsg, zap.Error(err)) return @@ -1327,20 +1343,28 @@ func (m *Messenger) downloadAndImportHistoryArchives(id types3.HexBytes, magnetl if downloadTaskInfo.TotalDownloadedArchivesCount > 0 { m.logger.Debug(fmt.Sprintf("downloaded %d of %d archives so far", downloadTaskInfo.TotalDownloadedArchivesCount, downloadTaskInfo.TotalArchivesCount)) } + m.archiveManager.UnseedHistoryArchive(id, archiveLink) return } - err = m.communitiesManager.UpdateLastSeenMagnetlink(id, magnetlink) + m.logger.Debug("[LogosStorage][download_and_import_history_archives] Updating last seen archiveLink", + zap.String("archiveLink", archiveLink)) + + err = m.communitiesManager.UpdateLastSeenArchiveLink(id, archiveLink) if err != nil { - m.logger.Error("couldn't update last seen magnetlink", zap.Error(err)) + m.logger.Error("couldn't update last seen archiveLink", zap.Error(err), zap.String("archiveLink", archiveLink)) } + m.archiveManager.PublishHistoryArchivesSeedingSignal(id) + err = m.checkIfIMemberOfCommunity(id) if err != nil { return } - err = m.importHistoryArchives(id, cancel) + m.logger.Debug("[LogosStorage][download_and_import_history_archives] Importing history archives now") + + err = m.importHistoryArchives(id, cancel, archiveLink) if err != nil { m.logger.Error("failed to import history archives", zap.Error(err)) m.config.messengerSignalsHandler.DownloadingHistoryArchivesFinished(types3.EncodeHex(id)) @@ -1635,11 +1659,14 @@ func (m *Messenger) HandleCommunityRequestToJoinResponse(ctx context.Context, st } } - magnetlink := requestToJoinResponseProto.MagnetUri - if m.archiveManager.IsReady() && communitySettings != nil && communitySettings.HistoryArchiveSupportEnabled && magnetlink != "" { + archiveLink := requestToJoinResponseProto.ArchiveLink + if m.archiveManager.IsStarted() && communitySettings != nil && communitySettings.HistoryArchiveSupportEnabled && archiveLink != "" { currentTask := m.archiveManager.GetHistoryArchiveDownloadTask(community.IDString()) - go func(currentTask *communities.HistoryArchiveDownloadTask) { + if err := m.communitiesManager.UpdateArchiveLinkMessageClock(requestToJoinResponseProto.CommunityId, requestToJoinResponseProto.Clock); err != nil { + return err + } + go func(currentTask *archivetypes.HistoryArchiveDownloadTask) { defer gocommon.LogOnPanic() // Cancel ongoing download/import task if currentTask != nil && !currentTask.IsCancelled() { @@ -1647,7 +1674,7 @@ func (m *Messenger) HandleCommunityRequestToJoinResponse(ctx context.Context, st currentTask.Waiter.Wait() } - task := &communities.HistoryArchiveDownloadTask{ + task := &archivetypes.HistoryArchiveDownloadTask{ CancelChan: make(chan struct{}), Waiter: *new(sync.WaitGroup), Cancelled: false, @@ -1656,11 +1683,14 @@ func (m *Messenger) HandleCommunityRequestToJoinResponse(ctx context.Context, st task.Waiter.Add(1) defer task.Waiter.Done() + defer m.archiveManager.RemoveHistoryArchiveDownloadTask(community.IDString()) m.shutdownWaitGroup.Add(1) defer m.shutdownWaitGroup.Done() - m.downloadAndImportHistoryArchives(community.ID(), magnetlink, task.CancelChan) + m.logger.Debug("[LogosStorage][handle_community_request_to_join_response] Starting download and import of history archives", zap.String("archiveLink", archiveLink)) + + m.downloadAndImportHistoryArchives(community.ID(), archiveLink, task.CancelChan) }(currentTask) } } @@ -3568,8 +3598,11 @@ func (m *Messenger) HandleSyncTrustedUser(ctx context.Context, state *ReceivedMe return nil } -func (m *Messenger) HandleCommunityMessageArchiveMagnetlink(ctx context.Context, state *ReceivedMessageState, message *protobuf.CommunityMessageArchiveMagnetlink, statusMessage *common.StatusMessage) error { - return m.HandleHistoryArchiveMagnetlinkMessage(state, state.CurrentMessageState.PublicKey, message.MagnetUri, message.Clock) + +func (m *Messenger) HandleCommunityMessageArchiveLink(ctx context.Context, state *ReceivedMessageState, message *protobuf.CommunityMessageArchiveLink, statusMessage *common.StatusMessage) error { + m.logger.Debug("[LogosStorage][HandleCommunityMessageArchiveLink] received CommunityMessageArchiveLink", zap.String("archiveLink", message.ArchiveLink)) + + return m.HandleHistoryArchiveLinkMessage(state, state.CurrentMessageState.PublicKey, message.ArchiveLink, message.Clock) } func (m *Messenger) addNewKeypairAddedOnPairedDeviceACNotification(keyUID string, response *MessengerResponse) error { diff --git a/protocol/protobuf/application_metadata_message.proto b/protocol/protobuf/application_metadata_message.proto index 1c7097dac35..bf0d0d812c0 100644 --- a/protocol/protobuf/application_metadata_message.proto +++ b/protocol/protobuf/application_metadata_message.proto @@ -50,7 +50,7 @@ message ApplicationMetadataMessage { SYNC_BOOKMARK = 40; SYNC_CLEAR_HISTORY = 41; SYNC_SETTING = 42; - COMMUNITY_MESSAGE_ARCHIVE_MAGNETLINK = 43; + COMMUNITY_MESSAGE_ARCHIVE_LINK = 43; SYNC_PROFILE_PICTURES = 44; SYNC_ACCOUNT = 45; ACCEPT_CONTACT_REQUEST = 46; diff --git a/protocol/protobuf/communities.proto b/protocol/protobuf/communities.proto index fc5ee8fefde..2bf44f6b6d7 100644 --- a/protocol/protobuf/communities.proto +++ b/protocol/protobuf/communities.proto @@ -112,7 +112,7 @@ message CommunityDescription { map chats = 6; repeated string ban_list = 7 [deprecated = true]; map categories = 8; - uint64 archive_magnetlink_clock = 9; + uint64 archive_link_clock = 9; CommunityAdminSettings admin_settings = 10; string intro_message = 11; string outro_message = 12; @@ -203,7 +203,7 @@ message CommunityRequestToJoinResponse { bool accepted = 3; bytes grant = 4; bytes community_id = 5; - string magnet_uri = 6; + string archive_link = 6; reserved 7, 8; // CommunityDescription protocol message with owner signature bytes community_description_protocol_message = 9; @@ -214,9 +214,9 @@ message CommunityRequestToLeave { bytes community_id = 2; } -message CommunityMessageArchiveMagnetlink { +message CommunityMessageArchiveLink { uint64 clock = 1; - string magnet_uri = 2; + string archive_link = 2; } message WakuMessage {