+1

go.mod

Reviewed

··· 54 54 golang.org/x/sys v0.30.0 55 55 google.golang.org/grpc v1.69.4 56 56 google.golang.org/protobuf v1.36.3 57 57 + pgregory.net/rapid v1.2.0 57 58 ) 58 59 59 60 require (

+2

go.sum

Reviewed

··· 576 576 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 577 577 mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= 578 578 mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= 579 579 + pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= 580 580 + pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= 579 581 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

+14 -10

index/shard_builder.go

Reviewed

··· 23 23 "net/url" 24 24 "os" 25 25 "path/filepath" 26 26 + "slices" 26 27 "sort" 27 28 "strings" 28 29 "text/template" 29 30 "time" 30 31 "unicode/utf8" 31 32 32 32 - "slices" 33 33 - 34 33 "github.com/sourcegraph/zoekt" 35 35 - "github.com/sourcegraph/zoekt/internal/languages" 34 34 + "github.com/sourcegraph/zoekt/languages" 36 35 ) 37 36 38 37 var _ = log.Println ··· 404 403 return 405 404 } 406 405 407 407 - if doc.SkipReason != SkipReasonNone { 408 408 - // If this document has been skipped, it's likely very large, or it's a non-code file like binary. 409 409 - // In this case, we just guess the language based on file name to avoid examining the contents. 410 410 - // Note: passing nil content is allowed by the go-enry contract (the underlying library we use here). 411 411 - doc.Language = languages.GetLanguage(doc.Name, nil) 412 412 - } else { 413 413 - doc.Language = languages.GetLanguage(doc.Name, doc.Content) 406 406 + // If this document has been skipped (doc.SkipReason != SkipReasonNone), it's 407 407 + // likely very large, or it's a non-code file like binary. In this case, we just 408 408 + // guess the language based on the file name to avoid examining the contents. 409 409 + // Note: passing nil content is allowed by the go-enry contract (the underlying 410 410 + // library we use here). 411 411 + var content []byte 412 412 + if doc.SkipReason == SkipReasonNone { 413 413 + content = doc.Content 414 414 + } 415 415 + langs := languages.GetLanguagesFromContent(doc.Name, content) 416 416 + if len(langs) > 0 { 417 417 + doc.Language = langs[0] 414 418 } 415 419 } 416 420

-74

internal/languages/language.go

Reviewed

··· 1 1 - // This file wraps the logic of go-enry (https://github.com/go-enry/go-enry) to support additional languages. 2 2 - // go-enry is based off of a package called Linguist (https://github.com/github/linguist) 3 3 - // and sometimes programming languages may not be supported by Linguist 4 4 - // or may take a while to get merged in and make it into go-enry. This wrapper 5 5 - // gives us flexibility to support languages in those cases. We list additional languages 6 6 - // in this file and remove them once they make it into Linguist and go-enry. 7 7 - // This logic is similar to what we have in the sourcegraph/sourcegraph repo, in the future 8 8 - // we plan to refactor both into a common library to share between the two repos. 9 9 - package languages 10 10 - 11 11 - import ( 12 12 - "path/filepath" 13 13 - "strings" 14 14 - 15 15 - "github.com/go-enry/go-enry/v2" 16 16 - ) 17 17 - 18 18 - var unsupportedByLinguistAliasMap = map[string]string{ 19 19 - // Extensions for the Apex programming language 20 20 - // See https://developer.salesforce.com/docs/atlas.en-us.apexcode.meta/apexcode/apex_dev_guide.htm 21 21 - "apex": "Apex", 22 22 - // Pkl Configuration Language (https://pkl-lang.org/) 23 23 - // Add to linguist on 6/7/24 24 24 - // can remove once go-enry package updates 25 25 - // to that linguist version 26 26 - "pkl": "Pkl", 27 27 - // Magik Language 28 28 - "magik": "Magik", 29 29 - } 30 30 - 31 31 - var unsupportedByLinguistExtensionToNameMap = map[string]string{ 32 32 - ".apex": "Apex", 33 33 - ".apxt": "Apex", 34 34 - ".apxc": "Apex", 35 35 - ".cls": "Apex", 36 36 - ".trigger": "Apex", 37 37 - // Pkl Configuration Language (https://pkl-lang.org/) 38 38 - ".pkl": "Pkl", 39 39 - // Magik Language 40 40 - ".magik": "Magik", 41 41 - } 42 42 - 43 43 - // getLanguagesByAlias is a replacement for enry.GetLanguagesByAlias 44 44 - // It supports languages that are missing in linguist 45 45 - func GetLanguageByAlias(alias string) (language string, ok bool) { 46 46 - language, ok = enry.GetLanguageByAlias(alias) 47 47 - if !ok { 48 48 - normalizedAlias := strings.ToLower(alias) 49 49 - language, ok = unsupportedByLinguistAliasMap[normalizedAlias] 50 50 - } 51 51 - 52 52 - return 53 53 - } 54 54 - 55 55 - // GetLanguage is a replacement for enry.GetLanguage 56 56 - // to find out the most probable language to return but includes support 57 57 - // for languages missing from linguist 58 58 - func GetLanguage(filename string, content []byte) (language string) { 59 59 - language = enry.GetLanguage(filename, content) 60 60 - 61 61 - // If go-enry failed to find language, fall back on our 62 62 - // internal check for languages missing in linguist 63 63 - if language == "" { 64 64 - ext := filepath.Ext(filename) 65 65 - normalizedExt := strings.ToLower(ext) 66 66 - if ext == "" { 67 67 - return 68 68 - } 69 69 - if lang, ok := unsupportedByLinguistExtensionToNameMap[normalizedExt]; ok { 70 70 - language = lang 71 71 - } 72 72 - } 73 73 - return 74 74 - }

-107

internal/languages/language_test.go

Reviewed

··· 1 1 - package languages 2 2 - 3 3 - import "testing" 4 4 - 5 5 - func TestGetLanguageByAlias(t *testing.T) { 6 6 - tests := []struct { 7 7 - name string 8 8 - alias string 9 9 - want string 10 10 - wantOk bool 11 11 - }{ 12 12 - { 13 13 - name: "empty alias", 14 14 - alias: "", 15 15 - want: "", 16 16 - wantOk: false, 17 17 - }, 18 18 - { 19 19 - name: "unknown alias", 20 20 - alias: "unknown", 21 21 - want: "", 22 22 - wantOk: false, 23 23 - }, 24 24 - { 25 25 - name: "supported alias", 26 26 - alias: "go", 27 27 - want: "Go", 28 28 - wantOk: true, 29 29 - }, 30 30 - { 31 31 - name: "unsupported by linguist alias", 32 32 - alias: "magik", 33 33 - want: "Magik", 34 34 - wantOk: true, 35 35 - }, 36 36 - { 37 37 - name: "unsupported by linguist alias normalized", 38 38 - alias: "mAgIk", 39 39 - want: "Magik", 40 40 - wantOk: true, 41 41 - }, 42 42 - { 43 43 - name: "apex example unsupported by linguist alias", 44 44 - alias: "apex", 45 45 - want: "Apex", 46 46 - wantOk: true, 47 47 - }, 48 48 - } 49 49 - 50 50 - for _, tt := range tests { 51 51 - t.Run(tt.name, func(t *testing.T) { 52 52 - got, ok := GetLanguageByAlias(tt.alias) 53 53 - if got != tt.want || ok != tt.wantOk { 54 54 - t.Errorf("GetLanguageByAlias(%q) = %q, %t, want %q, %t", tt.alias, got, ok, tt.want, tt.wantOk) 55 55 - } 56 56 - }) 57 57 - } 58 58 - } 59 59 - 60 60 - func TestGetLanguage(t *testing.T) { 61 61 - tests := []struct { 62 62 - name string 63 63 - filename string 64 64 - content []byte 65 65 - want string 66 66 - }{ 67 67 - { 68 68 - name: "empty filename", 69 69 - filename: "", 70 70 - content: []byte(""), 71 71 - want: "", 72 72 - }, 73 73 - { 74 74 - name: "unknown extension", 75 75 - filename: "file.unknown", 76 76 - content: []byte(""), 77 77 - want: "", 78 78 - }, 79 79 - { 80 80 - name: "supported extension", 81 81 - filename: "file.go", 82 82 - content: []byte("package main"), 83 83 - want: "Go", 84 84 - }, 85 85 - { 86 86 - name: "magik: unsupported by linguist extension", 87 87 - filename: "file.magik", 88 88 - content: []byte(""), 89 89 - want: "Magik", 90 90 - }, 91 91 - { 92 92 - name: "apex: unsupported by linguist extension", 93 93 - filename: "file.apxc", 94 94 - content: []byte(""), 95 95 - want: "Apex", 96 96 - }, 97 97 - } 98 98 - 99 99 - for _, tt := range tests { 100 100 - t.Run(tt.name, func(t *testing.T) { 101 101 - got := GetLanguage(tt.filename, tt.content) 102 102 - if got != tt.want { 103 103 - t.Errorf("GetLanguage(%q, %q) = %q, want %q", tt.filename, tt.content, got, tt.want) 104 104 - } 105 105 - }) 106 106 - } 107 107 - }

+15

languages/enry_vendored.go

Reviewed

··· 1 1 + package languages 2 2 + 3 3 + import "strings" 4 4 + 5 5 + // This file contains private functions 6 6 + // vendored from the go-enry codebase. 7 7 + 8 8 + // convertToAliasKey is vendored from go-enry to make sure 9 9 + // we're normalizing strings the same way. 10 10 + func convertToAliasKey(langName string) string { 11 11 + ak := strings.SplitN(langName, `,`, 2)[0] 12 12 + ak = strings.Replace(ak, ` `, `_`, -1) 13 13 + ak = strings.ToLower(ak) 14 14 + return ak 15 15 + }

+489

languages/extensions.go

Reviewed

··· 1 1 + package languages 2 2 + 3 3 + import ( 4 4 + "path/filepath" 5 5 + "slices" 6 6 + "strings" 7 7 + 8 8 + "github.com/go-enry/go-enry/v2" 9 9 + enrydata "github.com/go-enry/go-enry/v2/data" 10 10 + ) 11 11 + 12 12 + // GetLanguageByNameOrAlias returns the standardized name for 13 13 + // a language based on its name (in which case this is an identity operation) 14 14 + // or based on its alias, which is potentially an alternate name for 15 15 + // the language. 16 16 + // 17 17 + // Aliases are fully lowercase, and map N-1 to languages. 18 18 + // 19 19 + // For example, 20 20 + // 21 21 + // GetLanguageByNameOrAlias("ada") == "Ada", true 22 22 + // GetLanguageByNameOrAlias("ada95") == "Ada", true 23 23 + // 24 24 + // Historical note: This function was added for replacing usages of 25 25 + // enry.GetLanguageByAlias, which, unlike the name suggests, also 26 26 + // handles non-normalized names such as those with spaces. 27 27 + func GetLanguageByNameOrAlias(nameOrAlias string) (lang string, ok bool) { 28 28 + alias := convertToAliasKey(nameOrAlias) 29 29 + if lang, ok = unsupportedByEnryAliasMap[alias]; ok { 30 30 + return lang, true 31 31 + } 32 32 + 33 33 + return enry.GetLanguageByAlias(alias) 34 34 + } 35 35 + 36 36 + // GetLanguageExtensions returns the list of file extensions for a given 37 37 + // language. Returned extensions are always prefixed with a '.'. 38 38 + // 39 39 + // The returned slice will be empty iff the language is not known. 40 40 + // 41 41 + // Handles more languages than enry.GetLanguageExtensions. 42 42 + // 43 43 + // Mutually consistent with getLanguagesByExtension, see the tests 44 44 + // for the exact invariants. 45 45 + func GetLanguageExtensions(language string) []string { 46 46 + if langs, ok := unsupportedByEnryNameToExtensionMap[language]; ok { 47 47 + return langs 48 48 + } 49 49 + 50 50 + ignoreExts, isNiche := nicheExtensionUsages[language] 51 51 + // Force a copy to avoid accidentally modifying the global variable 52 52 + exts := slices.Clone(enry.GetLanguageExtensions(language)) 53 53 + for ext, lang := range sgExtraLangsForExts { // Map is tiny, so linear lookup is fine 54 54 + if language == lang { 55 55 + exts = append(exts, ext) 56 56 + } 57 57 + } 58 58 + if !isNiche { 59 59 + return exts 60 60 + } 61 61 + return slices.DeleteFunc(exts, func(ext string) bool { 62 62 + _, shouldIgnore := ignoreExts[ext] 63 63 + return shouldIgnore 64 64 + }) 65 65 + } 66 66 + 67 67 + // getLanguagesByExtension is a replacement for enry.GetLanguagesByExtension 68 68 + // to work around the following limitations: 69 69 + // - For some extensions which are overwhelmingly used by a certain file type 70 70 + // in practice, such as '.ts', '.md' and '.yaml', it returns ambiguous results. 71 71 + // - It does not provide any information about binary files. 72 72 + // - Some languages are not supported by enry yet (e.g. Magik) 73 73 + func getLanguagesByExtension(path string) (candidates []string, isLikelyBinaryFile bool) { 74 74 + // Lowercase extension before lookups to match enry's behavior. 75 75 + ext := strings.ToLower(filepath.Ext(path)) 76 76 + if ext == "" { 77 77 + return nil, false 78 78 + } 79 79 + if lang, ok := unsupportedByEnryExtensionToNameMap[ext]; ok { 80 80 + return []string{lang}, false 81 81 + } 82 82 + if _, ok := commonBinaryFileExtensions[ext[1:]]; ok { 83 83 + return nil, true 84 84 + } 85 85 + if lang, ok := overrideAmbiguousExtensionsMap[ext]; ok { 86 86 + return []string{lang}, false 87 87 + } 88 88 + candidates = enry.GetLanguagesByExtension(path, nil, nil) 89 89 + if extra, ok := sgExtraLangsForExts[ext]; ok { 90 90 + candidates = append(candidates, extra) 91 91 + } 92 92 + return candidates, false 93 93 + } 94 94 + 95 95 + var commonBinaryFileExtensions = func() map[string]struct{} { 96 96 + m := map[string]struct{}{} 97 97 + for _, s := range commonBinaryFileExtensionsList { 98 98 + m[s] = struct{}{} 99 99 + } 100 100 + return m 101 101 + }() 102 102 + 103 103 + var sgExtraLangsForExts = map[string]string{ 104 104 + ".c": "C++", 105 105 + // NOTE: Downstream code does linear lookups on this map, so 106 106 + // be careful if you're adding lots of entries here. 107 107 + } 108 108 + 109 109 + var sgExtraContentHeuristics = map[string]*enrydata.Heuristics{ 110 110 + ".c": enrydata.ContentHeuristics[".h"], 111 111 + } 112 112 + 113 113 + // overrideAmbiguousExtensionsMap represents extensions which are ambiguous according to 114 114 + // enry but not for Sourcegraph. 115 115 + var overrideAmbiguousExtensionsMap = map[string]string{ 116 116 + // Ignoring the uncommon usage of '.cs' for Smalltalk. 117 117 + ".cs": "C#", 118 118 + // The other languages are Filterscript, Forth, GLSL. Out of that, 119 119 + // Forth and GLSL commonly use other extensions. Ignore Filterscript 120 120 + // as it is niche. 121 121 + ".fs": "F#", 122 122 + // Ignoring the uncommon usage of '.html' for Ecmarkup. 123 123 + ".html": "HTML", 124 124 + // Ignoring other variants of JSON, such as OASv2-json and OASv3-json 125 125 + ".json": "JSON", 126 126 + // Not considering "GCC Machine Description". 127 127 + ".md": "Markdown", 128 128 + // The other main language using '.rs' is RenderScript, but that's deprecated. 129 129 + // See https://developer.android.com/guide/topics/renderscript/compute 130 130 + ".rs": "Rust", 131 131 + // In i18n contexts, there are XML files with '.ts' and '.tsx' extensions, 132 132 + // but we ignore those for now to avoid penalizing the common case. 133 133 + ".tsx": "TSX", 134 134 + ".ts": "TypeScript", 135 135 + // Ignoring "Adblock Filter List" and "Vim Help File". 136 136 + ".txt": "Text", 137 137 + // Ignoring other variants of YAML, such as MiniYAML, OASv2-yaml, OASv3-yaml. 138 138 + ".yaml": "YAML", 139 139 + ".yml": "YAML", 140 140 + // The PR adding Pkl support also listed another language called Pickle in 141 141 + // its heuristics, but doesn't have any real support for it. Just ignore 142 142 + // it. 143 143 + // https://github.com/github-linguist/linguist/pull/6730/files#diff-c2d2d7946540ab501a5ef7a7f54a57c530d8da599e41c2beb0fd2f5635d2fd50R539 144 144 + ".pkl": "Pkl", 145 145 + } 146 146 + 147 147 + // unsupportedByEnryExtensionToNameMap contains extension->name mappings 148 148 + // for languages not tracked by go-enry. 149 149 + var unsupportedByEnryExtensionToNameMap = map[string]string{ 150 150 + // Extensions for the Apex programming language 151 151 + // See https://developer.salesforce.com/docs/atlas.en-us.apexcode.meta/apexcode/apex_dev_guide.htm 152 152 + ".apex": "Apex", 153 153 + ".apxt": "Apex", 154 154 + ".apxc": "Apex", 155 155 + ".trigger": "Apex", 156 156 + ".magik": "Magik", 157 157 + } 158 158 + 159 159 + // nicheExtensionUsage keeps track of which (lang, extension) mappings 160 160 + // should not be considered. 161 161 + // 162 162 + // We cannot wholesale ignore these languages, as this list includes 163 163 + // languages like XML, but it can contain unusual extensions like '.tsx' 164 164 + // which we generally want to classify as TypeScript. 165 165 + var nicheExtensionUsages = func() map[string]map[string]struct{} { 166 166 + niche := map[string]map[string]struct{}{} 167 167 + considered := map[string]struct{}{} 168 168 + for _, lang := range overrideAmbiguousExtensionsMap { 169 169 + considered[lang] = struct{}{} 170 170 + } 171 171 + for ext := range overrideAmbiguousExtensionsMap { 172 172 + langs := enry.GetLanguagesByExtension("foo"+ext, nil, nil) 173 173 + for _, lang := range langs { 174 174 + if _, found := considered[lang]; !found { 175 175 + if m, hasMap := niche[lang]; hasMap { 176 176 + m[ext] = struct{}{} 177 177 + } else { 178 178 + niche[lang] = map[string]struct{}{ext: {}} 179 179 + } 180 180 + } 181 181 + } 182 182 + } 183 183 + for specialOverrideExt, lang := range unsupportedByEnryExtensionToNameMap { 184 184 + considered[lang] = struct{}{} 185 185 + langs := enry.GetLanguagesByExtension("foo"+specialOverrideExt, nil, nil) 186 186 + for _, lang := range langs { 187 187 + if _, found := considered[lang]; !found { 188 188 + if m, hasMap := niche[lang]; hasMap { 189 189 + m[specialOverrideExt] = struct{}{} 190 190 + } else { 191 191 + niche[lang] = map[string]struct{}{specialOverrideExt: {}} 192 192 + } 193 193 + } 194 194 + } 195 195 + } 196 196 + return niche 197 197 + }() 198 198 + 199 199 + // unsupportedByEnryNameToExtensionMap contains language->extension mappings 200 200 + // for languages not tracked by go-enry. 201 201 + var unsupportedByEnryNameToExtensionMap = reverseMap(unsupportedByEnryExtensionToNameMap) 202 202 + 203 203 + // unsupportedByEnryAliasMap maps alias -> language name for languages 204 204 + // not tracked by go-enry. 205 205 + var unsupportedByEnryAliasMap = func() map[string]string { 206 206 + out := map[string]string{} 207 207 + for _, lang := range unsupportedByEnryExtensionToNameMap { 208 208 + out[convertToAliasKey(lang)] = lang 209 209 + } 210 210 + return out 211 211 + }() 212 212 + 213 213 + func reverseMap(m map[string]string) map[string][]string { 214 214 + n := make(map[string][]string, len(m)) 215 215 + for k, v := range m { 216 216 + n[v] = append(n[v], k) 217 217 + } 218 218 + return n 219 219 + } 220 220 + 221 221 + // Source: https://github.com/sindresorhus/binary-extensions/blob/main/binary-extensions.json 222 222 + // License: https://github.com/sindresorhus/binary-extensions/blob/main/license 223 223 + // Replace the contents with 224 224 + // curl -L https://raw.githubusercontent.com/sindresorhus/binary-extensions/main/binary-extensions.json | jq '.[]' | awk '{print $1 ","}' 225 225 + // 226 226 + // Not adding a leading '.' here to make it easier to update/compare the list. 227 227 + var commonBinaryFileExtensionsList = []string{ 228 228 + "3dm", 229 229 + "3ds", 230 230 + "3g2", 231 231 + "3gp", 232 232 + "7z", 233 233 + "a", 234 234 + "aac", 235 235 + "adp", 236 236 + "afdesign", 237 237 + "afphoto", 238 238 + "afpub", 239 239 + "ai", 240 240 + "aif", 241 241 + "aiff", 242 242 + "alz", 243 243 + "ape", 244 244 + "apk", 245 245 + "appimage", 246 246 + "ar", 247 247 + "arj", 248 248 + "asf", 249 249 + "au", 250 250 + "avi", 251 251 + "bak", 252 252 + "baml", 253 253 + "bh", 254 254 + "bin", 255 255 + "bk", 256 256 + "bmp", 257 257 + "btif", 258 258 + "bz2", 259 259 + "bzip2", 260 260 + "cab", 261 261 + "caf", 262 262 + "cgm", 263 263 + "class", 264 264 + "cmx", 265 265 + "cpio", 266 266 + "cr2", 267 267 + "cur", 268 268 + "dat", 269 269 + "dcm", 270 270 + "deb", 271 271 + "dex", 272 272 + "djvu", 273 273 + "dll", 274 274 + "dmg", 275 275 + "dng", 276 276 + "doc", 277 277 + "docm", 278 278 + "docx", 279 279 + "dot", 280 280 + "dotm", 281 281 + "dra", 282 282 + "DS_Store", 283 283 + "dsk", 284 284 + "dts", 285 285 + "dtshd", 286 286 + "dvb", 287 287 + "dwg", 288 288 + "dxf", 289 289 + "ecelp4800", 290 290 + "ecelp7470", 291 291 + "ecelp9600", 292 292 + "egg", 293 293 + "eol", 294 294 + "eot", 295 295 + "epub", 296 296 + "exe", 297 297 + "f4v", 298 298 + "fbs", 299 299 + "fh", 300 300 + "fla", 301 301 + "flac", 302 302 + "flatpak", 303 303 + "fli", 304 304 + "flv", 305 305 + "fpx", 306 306 + "fst", 307 307 + "fvt", 308 308 + "g3", 309 309 + "gh", 310 310 + "gif", 311 311 + "graffle", 312 312 + "gz", 313 313 + "gzip", 314 314 + "h261", 315 315 + "h263", 316 316 + "h264", 317 317 + "icns", 318 318 + "ico", 319 319 + "ief", 320 320 + "img", 321 321 + "ipa", 322 322 + "iso", 323 323 + "jar", 324 324 + "jpeg", 325 325 + "jpg", 326 326 + "jpgv", 327 327 + "jpm", 328 328 + "jxr", 329 329 + "key", 330 330 + "ktx", 331 331 + "lha", 332 332 + "lib", 333 333 + "lvp", 334 334 + "lz", 335 335 + "lzh", 336 336 + "lzma", 337 337 + "lzo", 338 338 + "m3u", 339 339 + "m4a", 340 340 + "m4v", 341 341 + "mar", 342 342 + "mdi", 343 343 + "mht", 344 344 + "mid", 345 345 + "midi", 346 346 + "mj2", 347 347 + "mka", 348 348 + "mkv", 349 349 + "mmr", 350 350 + "mng", 351 351 + "mobi", 352 352 + "mov", 353 353 + "movie", 354 354 + "mp3", 355 355 + "mp4", 356 356 + "mp4a", 357 357 + "mpeg", 358 358 + "mpg", 359 359 + "mpga", 360 360 + "mxu", 361 361 + "nef", 362 362 + "npx", 363 363 + "numbers", 364 364 + "nupkg", 365 365 + "o", 366 366 + "odp", 367 367 + "ods", 368 368 + "odt", 369 369 + "oga", 370 370 + "ogg", 371 371 + "ogv", 372 372 + "otf", 373 373 + "ott", 374 374 + "pages", 375 375 + "pbm", 376 376 + "pcx", 377 377 + "pdb", 378 378 + "pdf", 379 379 + "pea", 380 380 + "pgm", 381 381 + "pic", 382 382 + "png", 383 383 + "pnm", 384 384 + "pot", 385 385 + "potm", 386 386 + "potx", 387 387 + "ppa", 388 388 + "ppam", 389 389 + "ppm", 390 390 + "pps", 391 391 + "ppsm", 392 392 + "ppsx", 393 393 + "ppt", 394 394 + "pptm", 395 395 + "pptx", 396 396 + "psd", 397 397 + "pya", 398 398 + "pyc", 399 399 + "pyo", 400 400 + "pyv", 401 401 + "qt", 402 402 + "rar", 403 403 + "ras", 404 404 + "raw", 405 405 + "resources", 406 406 + "rgb", 407 407 + "rip", 408 408 + "rlc", 409 409 + "rmf", 410 410 + "rmvb", 411 411 + "rpm", 412 412 + "rtf", 413 413 + "rz", 414 414 + "s3m", 415 415 + "s7z", 416 416 + "scpt", 417 417 + "sgi", 418 418 + "shar", 419 419 + "snap", 420 420 + "sil", 421 421 + "sketch", 422 422 + "slk", 423 423 + "smv", 424 424 + "snk", 425 425 + "so", 426 426 + "stl", 427 427 + "suo", 428 428 + "sub", 429 429 + "swf", 430 430 + "tar", 431 431 + "tbz", 432 432 + "tbz2", 433 433 + "tga", 434 434 + "tgz", 435 435 + "thmx", 436 436 + "tif", 437 437 + "tiff", 438 438 + "tlz", 439 439 + "ttc", 440 440 + "ttf", 441 441 + "txz", 442 442 + "udf", 443 443 + "uvh", 444 444 + "uvi", 445 445 + "uvm", 446 446 + "uvp", 447 447 + "uvs", 448 448 + "uvu", 449 449 + "viv", 450 450 + "vob", 451 451 + "war", 452 452 + "wav", 453 453 + "wax", 454 454 + "wbmp", 455 455 + "wdp", 456 456 + "weba", 457 457 + "webm", 458 458 + "webp", 459 459 + "whl", 460 460 + "wim", 461 461 + "wm", 462 462 + "wma", 463 463 + "wmv", 464 464 + "wmx", 465 465 + "woff", 466 466 + "woff2", 467 467 + "wrm", 468 468 + "wvx", 469 469 + "xbm", 470 470 + "xif", 471 471 + "xla", 472 472 + "xlam", 473 473 + "xls", 474 474 + "xlsb", 475 475 + "xlsm", 476 476 + "xlsx", 477 477 + "xlt", 478 478 + "xltm", 479 479 + "xltx", 480 480 + "xm", 481 481 + "xmind", 482 482 + "xpi", 483 483 + "xpm", 484 484 + "xwd", 485 485 + "xz", 486 486 + "z", 487 487 + "zip", 488 488 + "zipx", 489 489 + }

+205

languages/extensions_test.go

Reviewed

··· 1 1 + package languages 2 2 + 3 3 + import ( 4 4 + "slices" 5 5 + "strings" 6 6 + "testing" 7 7 + 8 8 + "github.com/go-enry/go-enry/v2" 9 9 + enrydata "github.com/go-enry/go-enry/v2/data" 10 10 + "github.com/stretchr/testify/require" 11 11 + ) 12 12 + 13 13 + // Languages/extensions that we don't want to regress 14 14 + var nonAmbiguousExtensionsCheck = map[string]string{ 15 15 + ".apex": "Apex", 16 16 + ".apxt": "Apex", 17 17 + ".apxc": "Apex", 18 18 + ".trigger": "Apex", 19 19 + ".js": "JavaScript", 20 20 + // Linguist removed JSX (but not TSX) as a separate language: 21 21 + // https://github.com/github-linguist/linguist/pull/5133 22 22 + ".jsx": "JavaScript", 23 23 + ".ts": "TypeScript", 24 24 + ".tsx": "TSX", 25 25 + ".py": "Python", 26 26 + ".rb": "Ruby", 27 27 + ".go": "Go", 28 28 + ".java": "Java", 29 29 + ".kt": "Kotlin", 30 30 + ".magik": "Magik", 31 31 + ".scala": "Scala", 32 32 + ".cs": "C#", 33 33 + ".fs": "F#", 34 34 + ".rs": "Rust", 35 35 + // ".c" is not included as we consider it ambiguous (C or C++) (SPLF-1309) 36 36 + ".cpp": "C++", 37 37 + ".cxx": "C++", 38 38 + ".html": "HTML", 39 39 + ".hpp": "C++", 40 40 + ".hxx": "C++", 41 41 + ".lua": "Lua", 42 42 + ".dart": "Dart", 43 43 + ".swift": "Swift", 44 44 + ".css": "CSS", 45 45 + ".json": "JSON", 46 46 + ".yml": "YAML", 47 47 + ".xml": "XML", 48 48 + ".pkl": "Pkl", 49 49 + } 50 50 + 51 51 + func TestGetLanguageByAlias_UnsupportedLanguages(t *testing.T) { 52 52 + for alias, name := range unsupportedByEnryAliasMap { 53 53 + resName, _ := GetLanguageByNameOrAlias(alias) 54 54 + require.Equal(t, name, resName, 55 55 + "maybe a typo in `unsupportedByEnryAliasMap`?") 56 56 + } 57 57 + } 58 58 + 59 59 + func TestGetLanguageByAlias_NonAmbiguousLanguages(t *testing.T) { 60 60 + for _, language := range nonAmbiguousExtensionsCheck { 61 61 + _, ok := GetLanguageByNameOrAlias(language) 62 62 + require.True(t, ok, 63 63 + "unable to find language %s in go-enry", language) 64 64 + } 65 65 + } 66 66 + 67 67 + func TestGetLanguageExtensions_UnsupportedExtensions(t *testing.T) { 68 68 + for language, exts := range unsupportedByEnryNameToExtensionMap { 69 69 + extensions := GetLanguageExtensions(language) 70 70 + for _, ext := range exts { 71 71 + require.Contains(t, extensions, ext, 72 72 + "maybe a typo in `unsupportedByEnryNameToExtensionMap`?") 73 73 + } 74 74 + } 75 75 + } 76 76 + 77 77 + func TestGetLanguageExtensions_NonAmbiguousExtensions(t *testing.T) { 78 78 + langMap := reverseMap(nonAmbiguousExtensionsCheck) 79 79 + for language, exts := range langMap { 80 80 + extensions := GetLanguageExtensions(language) 81 81 + for _, ext := range exts { 82 82 + require.Contains(t, extensions, ext, 83 83 + "If this test fails when updating enry, maybe `overrideAmbiguousExtensionsMap` needs updating") 84 84 + } 85 85 + } 86 86 + } 87 87 + 88 88 + func TestGetLanguagesByExtension_UnsupportedExtensions(t *testing.T) { 89 89 + for ext, language := range unsupportedByEnryExtensionToNameMap { 90 90 + filename := "foo" + ext 91 91 + languages, _ := getLanguagesByExtension(filename) 92 92 + require.Contains(t, languages, language, 93 93 + "maybe a typo in `unsupportedByEnryExtensionToNameMap`?") 94 94 + } 95 95 + } 96 96 + 97 97 + func TestGetLanguagesByExtension_OverrideExtensions(t *testing.T) { 98 98 + for ext, language := range overrideAmbiguousExtensionsMap { 99 99 + filename := "foo" + ext 100 100 + enryLangs := enry.GetLanguagesByExtension(filename, nil, nil) 101 101 + 102 102 + require.Contains(t, enryLangs, language, 103 103 + "maybe a typo in `overrideAmbiguousExtensionsMap`?") 104 104 + require.Greaterf(t, len(enryLangs), 1, 105 105 + "extension %v is not ambiguous according to enry, remove it from `overrideAmbiguousExtensionsMap`", 106 106 + ext) 107 107 + } 108 108 + } 109 109 + 110 110 + func TestGetLanguagesByExtension_NonAmbiguousExtensions(t *testing.T) { 111 111 + for ext, language := range nonAmbiguousExtensionsCheck { 112 112 + filename := "foo" + ext 113 113 + languages, isLikelyBinaryFile := getLanguagesByExtension(filename) 114 114 + require.False(t, isLikelyBinaryFile) 115 115 + require.Equal(t, []string{language}, languages, 116 116 + "If this test fails when updating enry, maybe `overrideAmbiguousExtensionsMap` needs updating") 117 117 + } 118 118 + } 119 119 + 120 120 + func TestGetLanguagesByExtension_BinaryExtensions(t *testing.T) { 121 121 + for _, ext := range []string{".png", ".jpg", ".gif"} { 122 122 + filename := "foo" + ext 123 123 + _, isLikelyBinary := getLanguagesByExtension(filename) 124 124 + require.Truef(t, isLikelyBinary, "filename: %v was not guessed to be binary;"+ 125 125 + "bug in extension matching logic in getLanguagesByExtension maybe?", 126 126 + filename) 127 127 + } 128 128 + } 129 129 + 130 130 + func TestExtensionsConsistency(t *testing.T) { 131 131 + for ext, overrideLang := range overrideAmbiguousExtensionsMap { 132 132 + filepath := "foo" + ext 133 133 + enryLangsForExt := enry.GetLanguagesByExtension(filepath, nil, nil) 134 134 + require.Containsf(t, enryLangsForExt, overrideLang, "overrideAmbiguousExtensionsMap maps extension %q to language %q but "+ 135 135 + "that mapping is not present in enry's list %v", ext, overrideLang, enryLangsForExt) 136 136 + require.Greaterf(t, len(enryLangsForExt), 1, "overrideAmbiguousExtensionsMap states that"+ 137 137 + "%q extension is ambiguous, but only found langs: %v", ext, enryLangsForExt) 138 138 + 139 139 + candidates, isLikelyBinary := getLanguagesByExtension(filepath) 140 140 + require.False(t, isLikelyBinary, "ambiguous files are all source code") 141 141 + require.True(t, len(candidates) == 1, "getLanguagesByExtension should respect overrideAmbiguousExtensionsMap") 142 142 + 143 143 + shouldBeIgnoredLangsForExt := slices.DeleteFunc(enryLangsForExt, func(s string) bool { 144 144 + return s == overrideLang 145 145 + }) 146 146 + for _, shouldBeIgnoredLang := range shouldBeIgnoredLangsForExt { 147 147 + ignoredExts, found := nicheExtensionUsages[shouldBeIgnoredLang] 148 148 + require.Truef(t, found, "expected lang: %q to have an entry in nicheExtensionUsages for consistency with GetLanguagesByExtension", shouldBeIgnoredLang) 149 149 + require.Truef(t, len(ignoredExts) >= 1, "sets in nicheExtensionUsages must be non-empty") 150 150 + 151 151 + nonNicheExts := GetLanguageExtensions(shouldBeIgnoredLang) 152 152 + for ignoredExt := range ignoredExts { 153 153 + require.Falsef(t, slices.Contains(nonNicheExts, ignoredExt), 154 154 + "GetLanguageExtensions should not return %q for lang %q for consistency with GetLanguagesByExtension", 155 155 + ignoredExt, shouldBeIgnoredLang) 156 156 + } 157 157 + } 158 158 + } 159 159 + } 160 160 + 161 161 + func TestExtensionsConsistency2(t *testing.T) { 162 162 + for lang := range enrydata.ExtensionsByLanguage { 163 163 + for _, ext := range GetLanguageExtensions(lang) { 164 164 + if strings.Count(ext, ".") > 1 { 165 165 + // Ignore unusual edge cases like .coffee.md for Literate CoffeeScript 166 166 + continue 167 167 + } 168 168 + langsByExt, isLikelyBinary := getLanguagesByExtension("foo" + ext) 169 169 + if !isLikelyBinary { 170 170 + require.Truef(t, slices.Contains(langsByExt, lang), 171 171 + "expected getLanguagesByExtension result %v to contain %q (extension: %q)", langsByExt, lang, ext) 172 172 + } 173 173 + } 174 174 + } 175 175 + } 176 176 + 177 177 + func TestUnsupportedByEnry(t *testing.T) { 178 178 + for lang := range unsupportedByEnryNameToExtensionMap { 179 179 + enry_extensions, found := enrydata.ExtensionsByLanguage[lang] 180 180 + if found { 181 181 + validateLanguageAgainstGoEnry(t, "unsupportedByEnryNameToExtensionMap", enry_extensions, lang) 182 182 + } 183 183 + } 184 184 + for _, lang := range unsupportedByEnryAliasMap { 185 185 + enry_extensions, found := enrydata.ExtensionsByLanguage[lang] 186 186 + if found { 187 187 + validateLanguageAgainstGoEnry(t, "unsupportedByEnryAliasMap", enry_extensions, lang) 188 188 + } 189 189 + } 190 190 + for _, lang := range unsupportedByEnryExtensionToNameMap { 191 191 + enry_extensions, found := enrydata.ExtensionsByLanguage[lang] 192 192 + if found { 193 193 + validateLanguageAgainstGoEnry(t, "unsupportedByEnryExtensionToNameMap", enry_extensions, lang) 194 194 + } 195 195 + } 196 196 + } 197 197 + 198 198 + func validateLanguageAgainstGoEnry(t *testing.T, name string, enryExtensions []string, lang string) { 199 199 + enryExtensions = slices.Clone(enryExtensions) 200 200 + slices.Sort(enryExtensions) 201 201 + sgExtensions := slices.Clone(unsupportedByEnryNameToExtensionMap[lang]) 202 202 + slices.Sort(sgExtensions) 203 203 + 204 204 + require.NotEqualf(t, enryExtensions, sgExtensions, "looks like language %q is supported by enry with the same extensions; remove it from %q", lang, name) 205 205 + }

+151

languages/languages.go

Reviewed

··· 1 1 + // Package languages provides enhanced language detection capabilities on top of 2 2 + // go-enry, with additional heuristics and mappings for better accuracy. 3 3 + package languages 4 4 + 5 5 + import ( 6 6 + "path/filepath" 7 7 + "slices" 8 8 + "strings" 9 9 + 10 10 + "github.com/go-enry/go-enry/v2" 11 11 + ) 12 12 + 13 13 + // Make sure all names are lowercase here, since they are normalized 14 14 + var enryLanguageMappings = map[string]string{ 15 15 + "c++": "cpp", 16 16 + "c#": "c_sharp", 17 17 + } 18 18 + 19 19 + // NormalizeLanguage converts the language name to lowercase and maps known 20 20 + // aliases to their canonical names. 21 21 + func NormalizeLanguage(filetype string) string { 22 22 + normalized := strings.ToLower(filetype) 23 23 + if mapped, ok := enryLanguageMappings[normalized]; ok { 24 24 + normalized = mapped 25 25 + } 26 26 + 27 27 + return normalized 28 28 + } 29 29 + 30 30 + // GetLanguages is a replacement for enry.GetLanguages which 31 31 + // avoids incorrect fallback behavior that is present in DefaultStrategies, 32 32 + // where it will misclassify '.h' header files as C when file contents 33 33 + // are not available. 34 34 + // 35 35 + // The content can be optionally passed via a callback instead of directly, so 36 36 + // that in the common case, the caller can avoid fetching the content. The full 37 37 + // content returned by getContent will be used for language detection. 38 38 + // 39 39 + // getContent is not called if the file is likely to be a binary file, 40 40 + // as enry only covers programming languages. 41 41 + // 42 42 + // The buffer provided by the getContent callback is not modified. 43 43 + // 44 44 + // Returns: 45 45 + // - An error if the getContent func returns an error 46 46 + // - An empty slice if language detection failed 47 47 + // - A single-element slice if the language was determined exactly 48 48 + // - A multi-element slice if the language was ambiguous. For example, 49 49 + // for simple `.h` files with just comments and macros, they may 50 50 + // be valid C, C++ or any of their derivative languages (e.g. Objective-C). 51 51 + func GetLanguages(path string, getContent func() ([]byte, error)) ([]string, error) { 52 52 + impl := func() ([]string, error) { 53 53 + langs := enry.GetLanguagesByFilename(path, nil, nil) 54 54 + if len(langs) == 1 { 55 55 + return langs, nil 56 56 + } 57 57 + newLangs, isLikelyBinaryFile := getLanguagesByExtension(path) 58 58 + if isLikelyBinaryFile { 59 59 + return nil, nil 60 60 + } 61 61 + switch len(newLangs) { 62 62 + case 0: 63 63 + break 64 64 + case 1: 65 65 + return newLangs, nil 66 66 + default: 67 67 + langs = newLangs 68 68 + } 69 69 + if getContent == nil { 70 70 + return langs, nil 71 71 + } 72 72 + content, err := getContent() 73 73 + if err != nil { 74 74 + return nil, err 75 75 + } 76 76 + if len(content) == 0 { 77 77 + return langs, nil 78 78 + } 79 79 + if enry.IsBinary(content) { 80 80 + return nil, nil 81 81 + } 82 82 + 83 83 + // enry doesn't expose a way to call GetLanguages with a specific set of 84 84 + // strategies, so just hand-roll that code here. 85 85 + var languages = langs 86 86 + for _, strategy := range []enry.Strategy{enry.GetLanguagesByModeline, getLanguagesByShebang, getLanguagesByContent, enry.GetLanguagesByClassifier} { 87 87 + candidates := strategy(path, content, languages) 88 88 + switch len(candidates) { 89 89 + case 0: 90 90 + continue 91 91 + case 1: 92 92 + return candidates, nil 93 93 + default: 94 94 + languages = candidates 95 95 + } 96 96 + } 97 97 + 98 98 + return languages, nil 99 99 + } 100 100 + 101 101 + langs, err := impl() 102 102 + return slices.Clone(langs), err 103 103 + } 104 104 + 105 105 + // GetLanguagesFromContent is a convenience wrapper around GetLanguages that 106 106 + // allows passing the content directly instead of a callback. 107 107 + func GetLanguagesFromContent(path string, content []byte) (langs []string) { 108 108 + // We can ignore the error here, because the callback will never return an error 109 109 + langs, _ = GetLanguages(path, func() ([]byte, error) { return content, nil }) 110 110 + return 111 111 + } 112 112 + 113 113 + // getLanguagesByContent is a wrapper for enry.GetLanguagesByContent. 114 114 + // 115 115 + // It applies additional heuristics for file extensions that need special handling. 116 116 + func getLanguagesByContent(path string, content []byte, candidates []string) []string { 117 117 + ext := strings.ToLower(filepath.Ext(path)) 118 118 + if heuristic, ok := sgExtraContentHeuristics[ext]; ok { 119 119 + return heuristic.Match(content) 120 120 + } 121 121 + return enry.GetLanguagesByContent(path, content, candidates) 122 122 + } 123 123 + 124 124 + // getLanguagesByShebang is a replacement for enry.GetLanguagesByShebang. 125 125 + // 126 126 + // The enry function considers non-programming languages such as 'Pod'/'Pod 6' 127 127 + // also for shebangs, so work around that. 128 128 + func getLanguagesByShebang(path string, content []byte, candidates []string) []string { 129 129 + languages := enry.GetLanguagesByShebang(path, content, candidates) 130 130 + if len(languages) == 2 { 131 131 + // See https://sourcegraph.com/github.com/go-enry/go-enry@40f2a1e5b90eec55c20441c2a5911dcfc298a447/-/blob/data/interpreter.go?L95-96 132 132 + if slices.Equal(languages, []string{"Perl", "Pod"}) { 133 133 + return []string{"Perl"} 134 134 + } 135 135 + if slices.Equal(languages, []string{"Pod 6", "Raku"}) { 136 136 + return []string{"Raku"} 137 137 + } 138 138 + } 139 139 + return slices.Clone(languages) 140 140 + } 141 141 + 142 142 + // IsLikelyVendoredFile returns true if the file is likely to be a vendored file. 143 143 + // 144 144 + // 1. This method is not 100% foolproof, as it relies on conventions 145 145 + // around file paths which may or may not be followed. 146 146 + // 2. The caller must not pass a directory path to this function 147 147 + // for short-circuiting, as there is no guarantee that if a path 148 148 + // p1 returns true, then Join(p1, p2) also returns true. 149 149 + func IsLikelyVendoredFile(path string) bool { 150 150 + return enry.IsVendor(path) 151 151 + }

+184

languages/languages_test.go

Reviewed

··· 1 1 + package languages 2 2 + 3 3 + import ( 4 4 + "testing" 5 5 + 6 6 + "github.com/go-enry/go-enry/v2" 7 7 + "github.com/stretchr/testify/require" 8 8 + "pgregory.net/rapid" 9 9 + ) 10 10 + 11 11 + var cppCapitalExtContent = `// Sample C++ file from the ROSE compiler project 12 12 + // Original source: https://github.com/rose-compiler/rose 13 13 + // This file is used for testing C++ language detection for files with .C extension 14 14 + // Attribution: ROSE Compiler Team - Lawrence Livermore National Laboratory 15 15 + 16 16 + #include "sage3basic.h" 17 17 + #include "rose_config.h" 18 18 + 19 19 + #include "SageTreeBuilder.h" 20 20 + #include "Jovial_to_ROSE_translation.h" 21 21 + #include "ModuleBuilder.h" 22 22 + 23 23 + #include <boost/optional/optional_io.hpp> 24 24 + #include <iostream> 25 25 + 26 26 + namespace Rose { 27 27 + namespace builder { 28 28 + 29 29 + using namespace Rose::Diagnostics; 30 30 + 31 31 + namespace SB = SageBuilder; 32 32 + namespace SI = SageInterface; 33 33 + namespace LT = LanguageTranslation; 34 34 + ` 35 35 + 36 36 + func TestGetLanguages(t *testing.T) { 37 37 + const matlabContent = "function [out] = square(x)\nout = x * x;\nend" 38 38 + const mathematicaContent = "f[x_] := x ^ 2\ng[y_] := f[y]" 39 39 + const cppContent = "namespace x { }" 40 40 + const cContent = "typedef struct { int x; } Int;" 41 41 + const emptyContent = "" 42 42 + 43 43 + testCases := []struct { 44 44 + path string 45 45 + content string 46 46 + expectedLanguages []string 47 47 + compareFirstOnly bool 48 48 + }{ 49 49 + {path: "perlscript", content: "#!/usr/bin/env perl\n$version = $ARGV[0];", expectedLanguages: []string{"Perl"}}, 50 50 + {path: "rakuscript", content: "#!/usr/bin/env perl6\n$version = $ARGV[0];", expectedLanguages: []string{"Raku"}}, 51 51 + {path: "ambiguous.h", content: emptyContent, expectedLanguages: []string{"C", "C++", "Objective-C"}}, 52 52 + {path: "cpp.h", content: cppContent, expectedLanguages: []string{"C++"}}, 53 53 + {path: "c.h", content: cContent, expectedLanguages: []string{"C"}}, 54 54 + {path: "matlab.m", content: matlabContent, expectedLanguages: []string{"MATLAB"}, compareFirstOnly: true}, 55 55 + {path: "mathematica.m", content: mathematicaContent, expectedLanguages: []string{"Mathematica"}, compareFirstOnly: true}, 56 56 + { 57 57 + path: "mathematica2.m", 58 58 + content: ` 59 59 + s := StringRiffle[{"a", "b", "c", "d", "e"}, ", "] 60 60 + Flatten[{{a, b}, {c, {d}, e}, {f, {g, h}}}] 61 61 + square[x_] := x ^ 2 62 62 + fourthpower[x_] := square[square[x]] 63 63 + `, 64 64 + expectedLanguages: []string{"Mathematica"}, 65 65 + compareFirstOnly: true, 66 66 + }, 67 67 + {path: "SageTreeBuilder.C", content: cppCapitalExtContent, expectedLanguages: []string{"C++"}}, 68 68 + // Ported cases from internal/languages TestGetLanguage 69 69 + {path: "", content: emptyContent, expectedLanguages: nil}, 70 70 + {path: "file.unknown", content: emptyContent, expectedLanguages: nil}, 71 71 + {path: "file.go", content: "package main", expectedLanguages: []string{"Go"}}, 72 72 + {path: "file.magik", content: emptyContent, expectedLanguages: []string{"Magik"}}, 73 73 + {path: "file.apxc", content: emptyContent, expectedLanguages: []string{"Apex"}}, 74 74 + // Check that we classify cls files by content and not just by extension 75 75 + {path: "tex.cls", content: `\DeclareOption*{}`, expectedLanguages: []string{"TeX", "Apex", "ObjectScript", "Visual Basic 6.0", "OpenEdge ABL", "VBA"}}, 76 76 + {path: "tex.cls", content: `public class HelloWorld {`, expectedLanguages: []string{"Apex", "Visual Basic 6.0", "TeX", "OpenEdge ABL", "ObjectScript", "VBA"}}, 77 77 + } 78 78 + 79 79 + for _, testCase := range testCases { 80 80 + var getContent func() ([]byte, error) 81 81 + if testCase.content != "" { 82 82 + getContent = func() ([]byte, error) { return []byte(testCase.content), nil } 83 83 + } 84 84 + gotLanguages, err := GetLanguages(testCase.path, getContent) 85 85 + require.NoError(t, err) 86 86 + if testCase.compareFirstOnly { 87 87 + require.Equal(t, testCase.expectedLanguages, gotLanguages[0:1]) 88 88 + continue 89 89 + } 90 90 + require.Equal(t, testCase.expectedLanguages, gotLanguages) 91 91 + } 92 92 + 93 93 + rapid.Check(t, func(t *rapid.T) { 94 94 + path := rapid.String().Draw(t, "path") 95 95 + content := rapid.SliceOfN(rapid.Byte(), 0, 100).Draw(t, "contents") 96 96 + require.NotPanics(t, func() { 97 97 + langs, err := GetLanguages(path, func() ([]byte, error) { return content, nil }) 98 98 + require.NoError(t, err) 99 99 + if len(langs) != 0 { 100 100 + for _, l := range langs { 101 101 + require.NotEqual(t, enry.OtherLanguage, l) 102 102 + } 103 103 + } 104 104 + }) 105 105 + }) 106 106 + 107 107 + rapid.Check(t, func(t *rapid.T) { 108 108 + baseName := "abcd" 109 109 + exts := []string{".h", ".m", ".unknown", ""} 110 110 + extGens := []*rapid.Generator[string]{} 111 111 + for _, ext := range exts { 112 112 + extGens = append(extGens, rapid.Just(ext)) 113 113 + } 114 114 + extension := rapid.OneOf(extGens...).Draw(t, "extension") 115 115 + path := baseName + extension 116 116 + contentGens := []*rapid.Generator[string]{} 117 117 + for _, content := range []string{cContent, cppContent, mathematicaContent, matlabContent, emptyContent} { 118 118 + contentGens = append(contentGens, rapid.Just(content)) 119 119 + } 120 120 + content := rapid.OneOf(contentGens...).Draw(t, "content") 121 121 + langs, err := GetLanguages(path, func() ([]byte, error) { 122 122 + return []byte(content), nil 123 123 + }) 124 124 + require.NoError(t, err) 125 125 + for _, lang := range langs { 126 126 + require.NotEqual(t, enry.OtherLanguage, lang) 127 127 + } 128 128 + }) 129 129 + } 130 130 + 131 131 + func TestGetLanguageByNameOrAlias(t *testing.T) { 132 132 + tests := []struct { 133 133 + name string 134 134 + alias string 135 135 + want string 136 136 + wantOk bool 137 137 + }{ 138 138 + { 139 139 + name: "empty alias", 140 140 + alias: "", 141 141 + want: "", 142 142 + wantOk: false, 143 143 + }, 144 144 + { 145 145 + name: "unknown alias", 146 146 + alias: "unknown", 147 147 + want: "", 148 148 + wantOk: false, 149 149 + }, 150 150 + { 151 151 + name: "supported alias", 152 152 + alias: "go", 153 153 + want: "Go", 154 154 + wantOk: true, 155 155 + }, 156 156 + { 157 157 + name: "unsupported by linguist alias", 158 158 + alias: "magik", 159 159 + want: "Magik", 160 160 + wantOk: true, 161 161 + }, 162 162 + { 163 163 + name: "unsupported by linguist alias normalized", 164 164 + alias: "mAgIk", 165 165 + want: "Magik", 166 166 + wantOk: true, 167 167 + }, 168 168 + { 169 169 + name: "apex example unsupported by linguist alias", 170 170 + alias: "apex", 171 171 + want: "Apex", 172 172 + wantOk: true, 173 173 + }, 174 174 + } 175 175 + 176 176 + for _, tt := range tests { 177 177 + t.Run(tt.name, func(t *testing.T) { 178 178 + got, ok := GetLanguageByNameOrAlias(tt.alias) 179 179 + if got != tt.want || ok != tt.wantOk { 180 180 + t.Errorf("GetLanguageByNameOrAlias(%q) = %q, %t, want %q, %t", tt.alias, got, ok, tt.want, tt.wantOk) 181 181 + } 182 182 + }) 183 183 + } 184 184 + }

+2 -2

query/parse.go

Reviewed

··· 22 22 23 23 "github.com/grafana/regexp" 24 24 25 25 - "github.com/sourcegraph/zoekt/internal/languages" 25 25 + "github.com/sourcegraph/zoekt/languages" 26 26 ) 27 27 28 28 var _ = log.Printf ··· 177 177 } 178 178 expr = q 179 179 case tokLang: 180 180 - canonical, ok := languages.GetLanguageByAlias(text) 180 180 + canonical, ok := languages.GetLanguageByNameOrAlias(text) 181 181 if !ok { 182 182 expr = &Const{false} 183 183 } else {