diff --git a/common.go b/common.go new file mode 100644 index 0000000..4eb59cf --- /dev/null +++ b/common.go @@ -0,0 +1,15 @@ +package main + +import ( + "strings" +) + +// isDiscordEmoji matches https://cdn.discordapp.com/emojis/503141595860959243.gif, and similar URLs/filenames +func isDiscordEmoji(link string) bool { + // always match discord emoji URLs, eg https://cdn.discordapp.com/emojis/340989430460317707.png + if strings.HasPrefix(link, discordEmojiBaseUrl) { + return true + } + + return false +} diff --git a/extract.go b/extract.go new file mode 100644 index 0000000..cf2d4bf --- /dev/null +++ b/extract.go @@ -0,0 +1,144 @@ +package main + +import ( + "fmt" + "net/url" +) + +func getDownloadLinks(inputURL string, channelID string, interactive bool) map[string]string { + if RegexpUrlTwitter.MatchString(inputURL) { + links, err := getTwitterUrls(inputURL) + if err != nil { + fmt.Println("twitter URL failed,", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlTwitterStatus.MatchString(inputURL) { + links, err := getTwitterStatusUrls(inputURL, channelID) + if err != nil { + fmt.Println("twitter status URL failed,", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlTistory.MatchString(inputURL) { + links, err := getTistoryUrls(inputURL) + if err != nil { + fmt.Println("tistory URL failed,", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlGfycat.MatchString(inputURL) { + links, err := getGfycatUrls(inputURL) + if err != nil { + fmt.Println("gfycat URL failed,", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlInstagram.MatchString(inputURL) { + links, err := getInstagramUrls(inputURL) + if err != nil { + fmt.Println("instagram URL failed,", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlImgurSingle.MatchString(inputURL) { + links, err := getImgurSingleUrls(inputURL) + if err != nil { + fmt.Println("imgur single URL failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlImgurAlbum.MatchString(inputURL) { + links, err := getImgurAlbumUrls(inputURL) + if err != nil { + fmt.Println("imgur album URL failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlGoogleDrive.MatchString(inputURL) { + links, err := getGoogleDriveUrls(inputURL) + if err != nil { + fmt.Println("google drive album URL failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlFlickrPhoto.MatchString(inputURL) { + links, err := getFlickrPhotoUrls(inputURL) + if err != nil { + fmt.Println("flickr photo URL failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlFlickrAlbum.MatchString(inputURL) { + links, err := getFlickrAlbumUrls(inputURL) + if err != nil { + fmt.Println("flickr album URL failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlFlickrAlbumShort.MatchString(inputURL) { + links, err := getFlickrAlbumShortUrls(inputURL) + if err != nil { + fmt.Println("flickr album short URL failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if RegexpUrlStreamable.MatchString(inputURL) { + links, err := getStreamableUrls(inputURL) + if err != nil { + fmt.Println("streamable URL failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + if DownloadTistorySites { + if RegexpUrlPossibleTistorySite.MatchString(inputURL) { + links, err := getPossibleTistorySiteUrls(inputURL) + if err != nil { + fmt.Println("checking for tistory site failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } + } + if RegexpUrlGoogleDriveFolder.MatchString(inputURL) { + if interactive { + links, err := getGoogleDriveFolderUrls(inputURL) + if err != nil { + fmt.Println("google drive folder URL failed, ", inputURL, ",", err) + } else if len(links) > 0 { + return skipDuplicateLinks(links, channelID, interactive) + } + } else { + fmt.Println("google drive folder only accepted in interactive channels") + } + } + + if !interactive && isDiscordEmoji(inputURL) { + fmt.Printf("skipped %s as it is a Discord emoji\n", inputURL) + return nil + } + + // try without queries + parsedURL, err := url.Parse(inputURL) + if err == nil { + parsedURL.RawQuery = "" + inputURLWithoutQueries := parsedURL.String() + if inputURLWithoutQueries != inputURL { + return getDownloadLinks(inputURLWithoutQueries, channelID, interactive) + } + } + + return map[string]string{inputURL: ""} +} diff --git a/main.go b/main.go index fe27513..67766d7 100644 --- a/main.go +++ b/main.go @@ -37,21 +37,6 @@ import ( var ( ChannelWhitelist map[string]string InteractiveChannelWhitelist map[string]string - RegexpUrlTwitter *regexp.Regexp - RegexpUrlTwitterStatus *regexp.Regexp - RegexpUrlTistory *regexp.Regexp - RegexpUrlTistoryWithCDN *regexp.Regexp - RegexpUrlGfycat *regexp.Regexp - RegexpUrlInstagram *regexp.Regexp - RegexpUrlImgurSingle *regexp.Regexp - RegexpUrlImgurAlbum *regexp.Regexp - RegexpUrlGoogleDrive *regexp.Regexp - RegexpUrlGoogleDriveFolder *regexp.Regexp - RegexpUrlPossibleTistorySite *regexp.Regexp - RegexpUrlFlickrPhoto *regexp.Regexp - RegexpUrlFlickrAlbum *regexp.Regexp - RegexpUrlFlickrAlbumShort *regexp.Regexp - RegexpUrlStreamable *regexp.Regexp dg *discordgo.Session DownloadTistorySites bool interactiveChannelLinkTemp map[string]string @@ -169,84 +154,9 @@ func main() { ) } - RegexpUrlTwitter, err = regexp.Compile(REGEXP_URL_TWITTER) + err = initRegex() if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlTwitterStatus, err = regexp.Compile(REGEXP_URL_TWITTER_STATUS) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlTistory, err = regexp.Compile(REGEXP_URL_TISTORY) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlTistoryWithCDN, err = regexp.Compile(REGEXP_URL_TISTORY_WITH_CDN) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlGfycat, err = regexp.Compile(REGEXP_URL_GFYCAT) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlInstagram, err = regexp.Compile(REGEXP_URL_INSTAGRAM) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlImgurSingle, err = regexp.Compile(REGEXP_URL_IMGUR_SINGLE) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlImgurAlbum, err = regexp.Compile(REGEXP_URL_IMGUR_ALBUM) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlGoogleDrive, err = regexp.Compile(REGEXP_URL_GOOGLEDRIVE) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlGoogleDriveFolder, err = regexp.Compile(REGEXP_URL_GOOGLEDRIVE_FOLDER) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlPossibleTistorySite, err = regexp.Compile(REGEXP_URL_POSSIBLE_TISTORY_SITE) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlFlickrPhoto, err = regexp.Compile(REGEXP_URL_FLICKR_PHOTO) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlFlickrAlbum, err = regexp.Compile(REGEXP_URL_FLICKR_ALBUM) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlStreamable, err = regexp.Compile(REGEXP_URL_STREAMABLE) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpUrlFlickrAlbumShort, err = regexp.Compile(REGEXP_URL_FLICKR_ALBUM_SHORT) - if err != nil { - fmt.Println("Regexp error", err) - return - } - RegexpFilename, err = regexp.Compile(REGEXP_FILENAME) - if err != nil { - fmt.Println("Regexp error", err) + fmt.Println("error initialising regex,", err.Error()) return } @@ -331,139 +241,6 @@ func messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) { } } -func getDownloadLinks(inputURL string, channelID string, interactive bool) map[string]string { - if RegexpUrlTwitter.MatchString(inputURL) { - links, err := getTwitterUrls(inputURL) - if err != nil { - fmt.Println("twitter URL failed,", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlTwitterStatus.MatchString(inputURL) { - links, err := getTwitterStatusUrls(inputURL, channelID) - if err != nil { - fmt.Println("twitter status URL failed,", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlTistory.MatchString(inputURL) { - links, err := getTistoryUrls(inputURL) - if err != nil { - fmt.Println("tistory URL failed,", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlGfycat.MatchString(inputURL) { - links, err := getGfycatUrls(inputURL) - if err != nil { - fmt.Println("gfycat URL failed,", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlInstagram.MatchString(inputURL) { - links, err := getInstagramUrls(inputURL) - if err != nil { - fmt.Println("instagram URL failed,", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlImgurSingle.MatchString(inputURL) { - links, err := getImgurSingleUrls(inputURL) - if err != nil { - fmt.Println("imgur single URL failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlImgurAlbum.MatchString(inputURL) { - links, err := getImgurAlbumUrls(inputURL) - if err != nil { - fmt.Println("imgur album URL failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlGoogleDrive.MatchString(inputURL) { - links, err := getGoogleDriveUrls(inputURL) - if err != nil { - fmt.Println("google drive album URL failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlFlickrPhoto.MatchString(inputURL) { - links, err := getFlickrPhotoUrls(inputURL) - if err != nil { - fmt.Println("flickr photo URL failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlFlickrAlbum.MatchString(inputURL) { - links, err := getFlickrAlbumUrls(inputURL) - if err != nil { - fmt.Println("flickr album URL failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlFlickrAlbumShort.MatchString(inputURL) { - links, err := getFlickrAlbumShortUrls(inputURL) - if err != nil { - fmt.Println("flickr album short URL failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if RegexpUrlStreamable.MatchString(inputURL) { - links, err := getStreamableUrls(inputURL) - if err != nil { - fmt.Println("streamable URL failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - if DownloadTistorySites { - if RegexpUrlPossibleTistorySite.MatchString(inputURL) { - links, err := getPossibleTistorySiteUrls(inputURL) - if err != nil { - fmt.Println("checking for tistory site failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } - } - if RegexpUrlGoogleDriveFolder.MatchString(inputURL) { - if interactive { - links, err := getGoogleDriveFolderUrls(inputURL) - if err != nil { - fmt.Println("google drive folder URL failed, ", inputURL, ",", err) - } else if len(links) > 0 { - return skipDuplicateLinks(links, channelID, interactive) - } - } else { - fmt.Println("google drive folder only accepted in interactive channels") - } - } - - // try without queries - parsedURL, err := url.Parse(inputURL) - if err == nil { - parsedURL.RawQuery = "" - inputURLWithoutQueries := parsedURL.String() - if inputURLWithoutQueries != inputURL { - return getDownloadLinks(inputURLWithoutQueries, channelID, interactive) - } - } - - return map[string]string{inputURL: ""} -} - func skipDuplicateLinks(linkList map[string]string, channelID string, interactive bool) map[string]string { if interactive == false { newList := make(map[string]string, 0) diff --git a/regex.go b/regex.go new file mode 100644 index 0000000..a760ae5 --- /dev/null +++ b/regex.go @@ -0,0 +1,93 @@ +package main + +import ( + "regexp" +) + +var ( + RegexpUrlTwitter *regexp.Regexp + RegexpUrlTwitterStatus *regexp.Regexp + RegexpUrlTistory *regexp.Regexp + RegexpUrlTistoryWithCDN *regexp.Regexp + RegexpUrlGfycat *regexp.Regexp + RegexpUrlInstagram *regexp.Regexp + RegexpUrlImgurSingle *regexp.Regexp + RegexpUrlImgurAlbum *regexp.Regexp + RegexpUrlGoogleDrive *regexp.Regexp + RegexpUrlGoogleDriveFolder *regexp.Regexp + RegexpUrlPossibleTistorySite *regexp.Regexp + RegexpUrlFlickrPhoto *regexp.Regexp + RegexpUrlFlickrAlbum *regexp.Regexp + RegexpUrlFlickrAlbumShort *regexp.Regexp + RegexpUrlStreamable *regexp.Regexp +) + +func initRegex() error { + var err error + RegexpUrlTwitter, err = regexp.Compile(REGEXP_URL_TWITTER) + if err != nil { + return err + } + RegexpUrlTwitterStatus, err = regexp.Compile(REGEXP_URL_TWITTER_STATUS) + if err != nil { + return err + } + RegexpUrlTistory, err = regexp.Compile(REGEXP_URL_TISTORY) + if err != nil { + return err + } + RegexpUrlTistoryWithCDN, err = regexp.Compile(REGEXP_URL_TISTORY_WITH_CDN) + if err != nil { + return err + } + RegexpUrlGfycat, err = regexp.Compile(REGEXP_URL_GFYCAT) + if err != nil { + return err + } + RegexpUrlInstagram, err = regexp.Compile(REGEXP_URL_INSTAGRAM) + if err != nil { + return err + } + RegexpUrlImgurSingle, err = regexp.Compile(REGEXP_URL_IMGUR_SINGLE) + if err != nil { + return err + } + RegexpUrlImgurAlbum, err = regexp.Compile(REGEXP_URL_IMGUR_ALBUM) + if err != nil { + return err + } + RegexpUrlGoogleDrive, err = regexp.Compile(REGEXP_URL_GOOGLEDRIVE) + if err != nil { + return err + } + RegexpUrlGoogleDriveFolder, err = regexp.Compile(REGEXP_URL_GOOGLEDRIVE_FOLDER) + if err != nil { + return err + } + RegexpUrlPossibleTistorySite, err = regexp.Compile(REGEXP_URL_POSSIBLE_TISTORY_SITE) + if err != nil { + return err + } + RegexpUrlFlickrPhoto, err = regexp.Compile(REGEXP_URL_FLICKR_PHOTO) + if err != nil { + return err + } + RegexpUrlFlickrAlbum, err = regexp.Compile(REGEXP_URL_FLICKR_ALBUM) + if err != nil { + return err + } + RegexpUrlStreamable, err = regexp.Compile(REGEXP_URL_STREAMABLE) + if err != nil { + return err + } + RegexpUrlFlickrAlbumShort, err = regexp.Compile(REGEXP_URL_FLICKR_ALBUM_SHORT) + if err != nil { + return err + } + RegexpFilename, err = regexp.Compile(REGEXP_FILENAME) + if err != nil { + return err + } + + return nil +} diff --git a/vars.go b/vars.go index bde1b63..21111bf 100644 --- a/vars.go +++ b/vars.go @@ -1,7 +1,7 @@ package main const ( - VERSION = "1.31.4" + VERSION = "1.32" DATABASE_DIR = "database" RELEASE_URL = "https://github.com/Seklfreak/discord-image-downloader-go/releases/latest" RELEASE_API_URL = "https://api.github.com/repos/Seklfreak/discord-image-downloader-go/releases/latest" @@ -25,4 +25,6 @@ const ( REGEXP_FILENAME = `^^[^/\\:*?"<>|]{1,150}\.[A-Za-z0-9]{2,4}$$` DEFAULT_CONFIG_FILE = "config.ini" + + discordEmojiBaseUrl = "https://cdn.discordapp.com/emojis/" )