1diff --git a/cmd/server/main.go b/cmd/server/main.go
2index 39987e5834e607f801155dab453d6ca7cffc8b0f..a3d5124e758937ca88d4b1cc89160e87f191bb39 100644
3--- a/cmd/server/main.go
4+++ b/cmd/server/main.go
5@@ -118,6 +118,7 @@ var (
6 fileScanner = scanner.NewFileScanner(mediaRepository, userRepository)
7 exifScanner = scanner.NewEXIFScanner(mediaRepository)
8 thumbnailScanner = scanner.NewThumbnailScanner(*cachePath, mediaRepository)
9+ albumScanner = scanner.NewAlbumScanner(mediaRepository)
10 )
11
12 // worker
13@@ -138,6 +139,11 @@ thumbnailScanner,
14 scheduler,
15 logrus.WithField("context", "thumbnail scanner"),
16 )
17+ albumWorker = worker.NewWorkerFromSerialProcessor[*repository.Media](
18+ albumScanner,
19+ scheduler,
20+ logrus.WithField("context", "thumbnail scanner"),
21+ )
22 )
23
24 pool := worker.NewWorkerPool()
25@@ -145,6 +151,7 @@ pool.AddWorker("http server", serverWorker)
26 pool.AddWorker("exif scanner", exifWorker)
27 pool.AddWorker("file scanner", fileWorker)
28 pool.AddWorker("thumbnail scanner", thumbnailWorker)
29+ pool.AddWorker("album scanner", albumWorker)
30
31 ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
32 defer stop()
33diff --git a/pkg/database/repository/media.go b/pkg/database/repository/media.go
34index 6f5b39b5326491e889cca59690c8a1fe00ff4fee..d6addbf7d2189814875083776bb38b422ed5aee3 100644
35--- a/pkg/database/repository/media.go
36+++ b/pkg/database/repository/media.go
37@@ -34,6 +34,10 @@ GPSLatitude *float64
38 GPSLongitude *float64
39 }
40
41+ Album struct {
42+ ID uint
43+ }
44+
45 MediaThumbnail struct {
46 Path string
47 }
48@@ -51,6 +55,17 @@ PathHash string
49 MIMEType string
50 }
51
52+ CreateAlbum struct {
53+ ParentID *uint
54+ Name string
55+ Path string
56+ }
57+
58+ CreateAlbumFile struct {
59+ MediaID uint
60+ AlbumID uint
61+ }
62+
63 MediaRepository interface {
64 Create(context.Context, *CreateMedia) error
65 Exists(context.Context, string) (bool, error)
66@@ -66,6 +81,12 @@
67 ListEmptyThumbnail(context.Context, *Pagination) ([]*Media, error)
68 GetThumbnail(context.Context, uint) (*MediaThumbnail, error)
69 CreateThumbnail(context.Context, uint, *MediaThumbnail) error
70+
71+ ListEmptyAlbums(context.Context, *Pagination) ([]*Media, error)
72+ ExistsAlbumByAbsolutePath(context.Context, string) (bool, error)
73+ GetAlbumByAbsolutePath(context.Context, string) (*Album, error)
74+ CreateAlbum(context.Context, *CreateAlbum) (*Album, error)
75+ CreateAlbumFile(context.Context, *CreateAlbumFile) error
76 }
77 )
78
79diff --git a/pkg/database/sql/media.go b/pkg/database/sql/media.go
80index e5ba517408453841bee462e8bc432879423564f4..59e39eebc37dd9a80b8e8f5e9a2452d827e88740 100644
81--- a/pkg/database/sql/media.go
82+++ b/pkg/database/sql/media.go
83@@ -48,6 +48,22 @@ MediaID uint
84 Media Media
85 }
86
87+ MediaAlbum struct {
88+ gorm.Model
89+ ParentID *uint
90+ Parent *MediaAlbum
91+ Name string
92+ Path string
93+ }
94+
95+ MediaAlbumFile struct {
96+ gorm.Model
97+ MediaID uint
98+ Media Media
99+ AlbumID uint
100+ Album MediaAlbum
101+ }
102+
103 MediaRepository struct {
104 db *gorm.DB
105 }
106@@ -55,13 +71,13 @@ )
107
108 var _ repository.MediaRepository = &MediaRepository{}
109
110-func (self *Media) ToModel() *repository.Media {
111+func (m *Media) ToModel() *repository.Media {
112 return &repository.Media{
113- ID: self.ID,
114- Path: self.Path,
115- PathHash: self.PathHash,
116- Name: self.Name,
117- MIMEType: self.MIMEType,
118+ ID: m.ID,
119+ Path: m.Path,
120+ PathHash: m.PathHash,
121+ Name: m.Name,
122+ MIMEType: m.MIMEType,
123 }
124 }
125
126@@ -83,6 +99,12 @@ Orientation: m.Orientation,
127 ExposureProgram: m.ExposureProgram,
128 GPSLatitude: m.GPSLatitude,
129 GPSLongitude: m.GPSLongitude,
130+ }
131+}
132+
133+func (a *MediaAlbum) ToModel() *repository.Album {
134+ return &repository.Album{
135+ ID: a.ID,
136 }
137 }
138
139@@ -329,3 +351,91 @@ }
140
141 return nil
142 }
143+
144+func (r *MediaRepository) ListEmptyAlbums(ctx context.Context, pagination *repository.Pagination) ([]*repository.Media, error) {
145+ medias := make([]*Media, 0)
146+ result := r.db.
147+ WithContext(ctx).
148+ Model(&Media{}).
149+ Joins("left join media_album_files on media.id = media_album_files.media_id").
150+ Where("media_album_files.media_id IS NULL").
151+ Offset(pagination.Page * pagination.Size).
152+ Limit(pagination.Size).
153+ Order("media.created_at DESC").
154+ Find(&medias)
155+
156+ if result.Error != nil {
157+ return nil, result.Error
158+ }
159+
160+ m := list.Map(medias, func(s *Media) *repository.Media {
161+ return s.ToModel()
162+ })
163+
164+ return m, nil
165+}
166+
167+func (m *MediaRepository) ExistsAlbumByAbsolutePath(ctx context.Context, path string) (bool, error) {
168+ var exists bool
169+ result := m.db.
170+ WithContext(ctx).
171+ Model(&MediaAlbum{}).
172+ Select("count(id) > 0").
173+ Where("path = ?", path).
174+ Find(&exists)
175+
176+ if result.Error != nil {
177+ return false, result.Error
178+ }
179+
180+ return exists, nil
181+}
182+
183+func (r *MediaRepository) GetAlbumByAbsolutePath(ctx context.Context, path string) (*repository.Album, error) {
184+ m := &MediaAlbum{}
185+ result := r.db.
186+ WithContext(ctx).
187+ Model(&MediaAlbum{}).
188+ Where("path = ?", path).
189+ Limit(1).
190+ Take(m)
191+
192+ if result.Error != nil {
193+ return nil, result.Error
194+ }
195+
196+ return m.ToModel(), nil
197+}
198+
199+func (m *MediaRepository) CreateAlbum(ctx context.Context, createAlbum *repository.CreateAlbum) (*repository.Album, error) {
200+ album := &MediaAlbum{
201+ ParentID: createAlbum.ParentID,
202+ Name: createAlbum.Name,
203+ Path: createAlbum.Path,
204+ }
205+
206+ result := m.db.
207+ WithContext(ctx).
208+ Create(album)
209+ if result.Error != nil {
210+ return nil, result.Error
211+ }
212+
213+ return album.ToModel(), nil
214+}
215+
216+func (m *MediaRepository) CreateAlbumFile(ctx context.Context, createAlbumFile *repository.CreateAlbumFile) error {
217+ albumFile := &MediaAlbumFile{
218+ MediaID: createAlbumFile.MediaID,
219+ AlbumID: createAlbumFile.AlbumID,
220+ }
221+
222+ result := m.db.
223+ WithContext(ctx).
224+ Create(albumFile)
225+ if result.Error != nil {
226+ return result.Error
227+ }
228+
229+ return nil
230+}
231diff --git a/pkg/database/sql/migration.go b/pkg/database/sql/migration.go
232index 076bf69da597bead7bd132bbad5473ad73c6be4e..73e4297679f7a640b4f726ac80c6a1cf116045f1 100644
233--- a/pkg/database/sql/migration.go
234+++ b/pkg/database/sql/migration.go
235@@ -9,6 +9,8 @@ &Settings{},
236 &Media{},
237 &MediaEXIF{},
238 &MediaThumbnail{},
239+ &MediaAlbum{},
240+ &MediaAlbumFile{},
241 } {
242 if err := db.AutoMigrate(m); err != nil {
243 return err
244diff --git a/pkg/list/list.go b/pkg/list/list.go
245index ff259f7f3dbf057e373eb36ecac89f97f6f0c6fd..dfc3fb7bce58752364d08c552a45be6eb47b047a 100644
246--- a/pkg/list/list.go
247+++ b/pkg/list/list.go
248@@ -7,3 +7,28 @@ result = append(result, fun(s))
249 }
250 return result
251 }
252+
253+type Pair[T, U any] struct {
254+ Left T
255+ Right U
256+}
257+
258+func Zip[T, U any](left []T, right []U) []Pair[T, U] {
259+ // pick the array with the smaller length
260+ l := len(left)
261+ if len(left) > len(right) {
262+ l = len(right)
263+ }
264+
265+ pairs := make([]Pair[T, U], len(left))
266+ for i := 0; i < l; i++ {
267+ pairs[i] = Pair[T, U]{left[i], right[i]}
268+ }
269+ return pairs
270+}
271+
272+func Revert[T any](s []T) {
273+ for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
274+ s[i], s[j] = s[j], s[i]
275+ }
276+}
277diff --git a/pkg/worker/list_processor.go b/pkg/worker/list_processor.go
278index c060583bc9ebb22d2a0183deffe1ba42f7111acd..02817c94e8074b392580606b842fb25c0c679d2d 100644
279--- a/pkg/worker/list_processor.go
280+++ b/pkg/worker/list_processor.go
281@@ -20,7 +20,7 @@ OnFail[T any] interface {
282 OnFail(context.Context, T, error)
283 }
284
285- BatchProcessor[T any] interface {
286+ ListProcessor[T any] interface {
287 Query(context.Context) ([]T, error)
288 Process(context.Context, T) error
289 }
290@@ -32,18 +32,36 @@ scheduler *Scheduler
291 }
292
293 batchProcessorWorker[T any] struct {
294- batchProcessor BatchProcessor[T]
295+ batchProcessor ListProcessor[T]
296+ logrus *logrus.Entry
297+ scheduler *Scheduler
298+ }
299+
300+ serialProcessorWorker[T any] struct {
301+ batchProcessor ListProcessor[T]
302 logrus *logrus.Entry
303 scheduler *Scheduler
304 }
305 )
306
307 func NewWorkerFromBatchProcessor[T any](
308- batchProcessor BatchProcessor[T],
309+ batchProcessor ListProcessor[T],
310 scheduler *Scheduler,
311 logrus *logrus.Entry,
312 ) Worker {
313 return &batchProcessorWorker[T]{
314+ batchProcessor: batchProcessor,
315+ scheduler: scheduler,
316+ logrus: logrus,
317+ }
318+}
319+
320+func NewWorkerFromSerialProcessor[T any](
321+ batchProcessor ListProcessor[T],
322+ scheduler *Scheduler,
323+ logrus *logrus.Entry,
324+) Worker {
325+ return &serialProcessorWorker[T]{
326 batchProcessor: batchProcessor,
327 scheduler: scheduler,
328 logrus: logrus,
329@@ -102,6 +120,41 @@ }(v)
330 }
331
332 wg.Wait()
333+ }
334+}
335+
336+func (l *serialProcessorWorker[T]) Start(ctx context.Context) error {
337+ for {
338+ values, err := l.batchProcessor.Query(ctx)
339+ if err != nil {
340+ return err
341+ }
342+
343+ select {
344+ case <-ctx.Done():
345+ return ctx.Err()
346+ default:
347+ }
348+
349+ if len(values) == 0 {
350+ return nil
351+ }
352+ for _, v := range values {
353+ select {
354+ case <-ctx.Done():
355+ return ctx.Err()
356+ default:
357+ }
358+
359+ l.scheduler.Take()
360+ if err := l.batchProcessor.Process(ctx, v); err != nil && !errors.Is(err, context.Canceled) {
361+ l.logrus.WithError(err).Error("Error processing batch")
362+ if failure, ok := l.batchProcessor.(OnFail[T]); ok {
363+ failure.OnFail(ctx, v, err)
364+ }
365+ }
366+ l.scheduler.Return()
367+ }
368 }
369 }
370
371diff --git a/pkg/worker/scanner/album_scanner.go b/pkg/worker/scanner/album_scanner.go
372new file mode 100644
373index 0000000000000000000000000000000000000000..618a184d157d6fba7abfcf3e64823107ddba0583
374--- /dev/null
375+++ b/pkg/worker/scanner/album_scanner.go
376@@ -0,0 +1,98 @@
377+package scanner
378+
379+import (
380+ "context"
381+ "os"
382+ "path"
383+ "path/filepath"
384+ "strings"
385+
386+ "git.sr.ht/~gabrielgio/img/pkg/database/repository"
387+ "git.sr.ht/~gabrielgio/img/pkg/worker"
388+)
389+
390+type (
391+ AlbumScanner struct {
392+ repository repository.MediaRepository
393+ }
394+)
395+
396+var _ worker.ListProcessor[*repository.Media] = &AlbumScanner{}
397+
398+func NewAlbumScanner(repository repository.MediaRepository) *AlbumScanner {
399+ return &AlbumScanner{
400+ repository: repository,
401+ }
402+}
403+
404+func (e *AlbumScanner) Query(ctx context.Context) ([]*repository.Media, error) {
405+ return e.repository.ListEmptyAlbums(ctx, &repository.Pagination{
406+ Page: 0,
407+ Size: 100,
408+ })
409+}
410+
411+// This process will optmize for file over folder, which means it will assume that there will be
412+// more file then folder in the overall library.
413+// So will try to make as cheap as possible to look for fetching many files in a folder
414+// meaning it will start from checking from left to right in the path since it will assume
415+// that the path to that point has been registered already, resulting in a single lookup for the media
416+func (e *AlbumScanner) Process(ctx context.Context, m *repository.Media) error {
417+ // we don't need the name of the file, only its path
418+ filePath, _ := path.Split(m.Path)
419+
420+ parts := strings.Split(filePath, string(os.PathSeparator))
421+
422+ subPaths := FanInwards(parts)
423+ album, err := e.GetAndCreateNestedAlbuns(ctx, subPaths)
424+ if err != nil {
425+ return err
426+ }
427+
428+ return e.repository.CreateAlbumFile(ctx, &repository.CreateAlbumFile{
429+ MediaID: m.ID,
430+ AlbumID: album.ID,
431+ })
432+}
433+
434+func (e *AlbumScanner) GetAndCreateNestedAlbuns(ctx context.Context, paths []string) (*repository.Album, error) {
435+ if len(paths) == 1 {
436+ // end of trail, we should create a album without parent
437+ return e.repository.CreateAlbum(ctx, &repository.CreateAlbum{
438+ ParentID: nil,
439+ Name: filepath.Base(paths[0]),
440+ Path: paths[0],
441+ })
442+ }
443+
444+ exists, err := e.repository.ExistsAlbumByAbsolutePath(ctx, paths[0])
445+ if err != nil {
446+ return nil, err
447+ }
448+
449+ if exists {
450+ return e.repository.GetAlbumByAbsolutePath(ctx, paths[0])
451+ }
452+
453+ //album does not exist, create it and get its parent id
454+ a, err := e.GetAndCreateNestedAlbuns(ctx, paths[1:])
455+ if err != nil {
456+ return nil, err
457+ }
458+
459+ return e.repository.CreateAlbum(ctx, &repository.CreateAlbum{
460+ ParentID: &a.ID,
461+ Name: filepath.Base(paths[0]),
462+ Path: paths[0],
463+ })
464+
465+}
466+
467+func FanInwards(paths []string) []string {
468+ result := make([]string, 0, len(paths))
469+ for i := (len(paths) - 1); i >= 0; i-- {
470+ subPaths := paths[0:i]
471+ result = append(result, path.Join(subPaths...))
472+ }
473+ return result
474+}
475diff --git a/pkg/worker/scanner/exif_scanner.go b/pkg/worker/scanner/exif_scanner.go
476index 47d717fc3d48eaf99945a504d86a76a9ef21955b..da63c0b790a1491c49c7b0ad3b8cf3bdcc91baa3 100644
477--- a/pkg/worker/scanner/exif_scanner.go
478+++ b/pkg/worker/scanner/exif_scanner.go
479@@ -15,7 +15,7 @@ repository repository.MediaRepository
480 }
481 )
482
483-var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{}
484+var _ worker.ListProcessor[*repository.Media] = &EXIFScanner{}
485
486 func NewEXIFScanner(repository repository.MediaRepository) *EXIFScanner {
487 return &EXIFScanner{
488diff --git a/pkg/worker/scanner/thumbnail_scanner.go b/pkg/worker/scanner/thumbnail_scanner.go
489index 9f75464fa07512fa655375a7a9f9a4bef011cdef..6446c53271d918ffe3ee24fa68950db8a3bd6fac 100644
490--- a/pkg/worker/scanner/thumbnail_scanner.go
491+++ b/pkg/worker/scanner/thumbnail_scanner.go
492@@ -19,7 +19,7 @@ cachePath string
493 }
494 )
495
496-var _ worker.BatchProcessor[*repository.Media] = &EXIFScanner{}
497+var _ worker.ListProcessor[*repository.Media] = &EXIFScanner{}
498
499 func NewThumbnailScanner(cachePath string, repository repository.MediaRepository) *ThumbnailScanner {
500 return &ThumbnailScanner{