imgur-rescue-project/irp2bolt/main.go

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"go.etcd.io/bbolt"
)

type ImgurSingleMedia struct {
	ID          string    `json:"id"`
	Title       string    `json:"title"`
	Description string    `json:"description"`
	MimeType    string    `json:"mime_type"`
	CreatedAt   time.Time `json:"created_at"` // e.g. 2013-10-28T02:37:02Z

	Width     int64  `json:"width"`
	Height    int64  `json:"height"`
	Extension string `json:"ext"`
}

func (ism ImgurSingleMedia) InventName() string {
	ret := ism.Title
	if len(ism.Description) > 0 {
		if len(ret) > 0 {
			ret += " - "
		}
		ret += ism.Description
	}

	if len(ret) == 0 {
		// No name/description in either gallery nor in first image
		// Guess we just name it after the ID
		ret = ism.ID
	}

	ret += "." + ism.Extension

	return ret
}

type ImgurInfo struct {
	ID          string             `json:"id"`
	Title       string             `json:"title"`
	Description string             `json:"description"`
	CreatedAt   time.Time          `json:"created_at"`
	Media       []ImgurSingleMedia `json:"media"`
}

func (i ImgurInfo) AlbumJson() []byte {
	arr := make([]string, 0, len(i.Media))
	for _, m := range i.Media {
		arr = append(arr, m.ID)
	}

	bb, err := json.Marshal(arr)
	if err != nil {
		panic(err)
	}

	return bb
}

func (i ImgurInfo) InventName() string {
	ret := i.Title

	if len(i.Description) > 0 {
		if len(ret) > 0 {
			ret += " - "
		}
		ret += i.Description
	}

	if len(ret) > 0 {
		return ret // Title + description is pretty good for an album
	}

	if len(i.Media) > 0 {
		// Describe this album based on the first media instead
		return i.Media[0].InventName()
	}

	// No name/description in either gallery nor in first image
	// Guess we just name it after the ID
	return i.ID
}

type ContentedMetadata struct {
	FileHash   string
	FileSize   int64
	UploadTime time.Time
	UploadIP   string
	Filename   string
	MimeType   string
}

func main() {
	db, err := bbolt.Open(fmt.Sprintf("output-%d.db", time.Now().Unix()), 0644, bbolt.DefaultOptions)
	if err != nil {
		panic(err)
	}

	err = db.Update(func(tx *bbolt.Tx) error {

		bb, err := tx.CreateBucketIfNotExists([]byte(`METADATA`))
		if err != nil {
			panic(err)
		}

		//
		// Media
		//

		media, err := os.ReadDir("../metadata.media")
		if err != nil {
			panic(err)
		}

		var addMediaCount int64 = 0

		for _, mediaInfo := range media {
			infoJson, err := ioutil.ReadFile("../metadata.media/" + mediaInfo.Name())
			if err != nil {
				panic(err)
			}

			var info ImgurInfo
			err = json.Unmarshal(infoJson, &info)
			if err != nil {
				panic(err)
			}

			if len(info.Media) != 1 {
				panic(err)
			}

			// Ensure image file exists
			finfo, err := os.Stat("../images/" + mediaInfo.Name())
			if err != nil {
				log.Printf("Missing image %s for media %s, skipping", mediaInfo.Name(), mediaInfo.Name())
				continue
				// panic(err)
			}

			cinfoBytes, err := json.Marshal(ContentedMetadata{
				FileHash:   mediaInfo.Name(),
				FileSize:   finfo.Size(),
				UploadTime: info.CreatedAt,
				UploadIP:   "n/a",
				Filename:   info.InventName(),
				MimeType:   info.Media[0].MimeType,
			})
			if err != nil {
				panic(err)
			}

			err = bb.Put([]byte(mediaInfo.Name()), cinfoBytes)
			if err != nil {
				panic(err)
			}

			addMediaCount += 1
		}

		log.Printf("Added %d media entries OK", addMediaCount)

		//
		// Albums
		//

		albums, err := os.ReadDir("../metadata.albums")
		if err != nil {
			panic(err)
		}

		var addAlbumCount int64 = 0
		var addAlbumMediaCount int64 = 0
		var albumsWithNoImagesCount int64 = 0

		for _, albuminfo := range albums {
			infoJson, err := ioutil.ReadFile("../metadata.albums/" + albuminfo.Name())
			if err != nil {
				panic(err)
			}

			var info ImgurInfo
			err = json.Unmarshal(infoJson, &info)
			if err != nil {
				panic(err)
			}

			if len(info.Media) == 0 {
				log.Printf("Album '%s' contains no images, allowing anyway", albuminfo.Name())
				albumsWithNoImagesCount += 1
			}

			// Add gallery entries for each of the media elements

			for _, mediaInfo := range info.Media {

				// Ensure image file exists
				finfo, err := os.Stat("../images/" + mediaInfo.ID)
				if err != nil {
					log.Printf("Missing image %s for album %s, skipping", mediaInfo.ID, albuminfo.Name())
					continue
					// panic(err)
				}

				cinfoBytes, err := json.Marshal(ContentedMetadata{
					FileHash:   mediaInfo.ID,
					FileSize:   finfo.Size(),
					UploadTime: info.CreatedAt,
					UploadIP:   "n/a",
					Filename:   mediaInfo.InventName(),
					MimeType:   mediaInfo.MimeType,
				})
				if err != nil {
					panic(err)
				}

				err = bb.Put([]byte(mediaInfo.ID), cinfoBytes)
				if err != nil {
					panic(err)
				}

				addAlbumMediaCount += 1

			}

			// Add album entry for the overall album
			albumHash := `a/` + albuminfo.Name() // Use a/ prefix. This can't naturally happen in contented's filehash algorithm
			albumJson := info.AlbumJson()
			err = ioutil.WriteFile(albumHash, albumJson, 0644) // a/ subdirectory must exist
			if err != nil {
				panic(err)
			}

			cinfoBytes, err := json.Marshal(ContentedMetadata{
				FileHash:   albumHash,
				FileSize:   int64(len(albumJson)),
				UploadTime: info.CreatedAt,
				UploadIP:   "n/a",
				Filename:   info.InventName(),
				MimeType:   "contented/album",
			})
			if err != nil {
				panic(err)
			}

			err = bb.Put([]byte(albumHash), cinfoBytes)
			if err != nil {
				panic(err)
			}

			addAlbumCount += 1
		}

		log.Printf("Added %d album entries OK with %d additional image entries", addAlbumCount, addMediaCount)

		log.Printf("There are %d albums with no images", albumsWithNoImagesCount)

		// Fully imported
		return nil
	})
	if err != nil {
		panic(err)
	}
}