Module:Dependencies/Parse

Documentation for this module may be created at Module:Dependencies/Parse/doc

require('strict')

--- @module Dependencies/Parse
--- Reads a page's own source and returns what it depends on: the wiki modules a
--- Lua module requires or loads, the module functions a template invokes, the
--- TemplateStyles stylesheets either one loads, and the image files it names.
--- Pure: the caller passes the source text.

local p = {}

--- Libraries a module may require that are not wiki pages.
local BUILTINS = { libraryUtil = true, strict = true, bit32 = true }

--- Extensions a name must end in to count as a file.
local FILE_EXTENSIONS = { svg = true, png = true, jpg = true, jpeg = true, gif = true, webp = true }

--- A URL on this wiki's file storage, up to the end of its path. The path is
--- `a/ab/<File>` (or `thumb/a/ab/<File>/…`), `ab` being the start of the md5 of
--- the file's name.
local MEDIA_URL = '%a*:?//media%.starcitizen%.tools/[^%s"\'%)%[%]|{}<>]+'

--- A run of wikitext between the characters that cannot be part of a file name
--- and the syntax around one: `[[File:X.svg|20px]]` yields `File`, `X.svg`, `20px`.
local SEGMENT = '[^|=%[%]{}<>\n:"]+'

--- A `string.format` directive, which stands for a value filled in at runtime.
local FORMAT_DIRECTIVE = '%%[-%d%.]*[sdi]'

--- The calls that name a module, and the list each one feeds.
local CALLS = {
	{ pattern = 'require', list = 'requires' },
	{ pattern = 'mw%.loadData', list = 'loads' },
	{ pattern = 'mw%.loadJsonData', list = 'loads' },
}

--- A Lua pattern matching `word` in any letter case.
--- @param word string
--- @return string
local function caseless(word)
	return (word:gsub('%a', function(letter)
		return '[' .. letter:upper() .. letter:lower() .. ']'
	end))
end

--- The start of a `]-' .. caseless('src') .. '%s*=%s*'

--- @param text string wikitext
--- @return string
local function stripHtmlComments(text)
	return (text:gsub('<!%-%-.-%-%->', ''))
end

--- @param set table<string, boolean>
--- @return string[]
local function sortedKeys(set)
	local list = {}
	for key in pairs(set) do
		list[#list + 1] = key
	end
	table.sort(list)
	return list
end

--- The string a single literal argument holds, or nil. A value containing its
--- own quote character is the concatenation of two literals, not one.
--- @param arg string
--- @return string|nil
local function literal(arg)
	local quote, value = arg:match('^%s*([\'"])(.-)%1%s*$')
	if value == nil or value:find(quote, 1, true) then
		return nil
	end
	return value
end

--- The string literal assigned to a variable elsewhere in the source, or nil.
--- @param src string
--- @param name string an identifier
--- @return string|nil
local function variableValue(src, name)
	local quote, value = src:match('[^%w_]' .. name .. '%s*=%s*([\'"])(.-)%1')
	if quote == nil then
		return nil
	end
	return value
end

--- Joins a concatenation's literal parts, with `…` for every non-literal part.
--- @param arg string
--- @return string
local function concatenation(arg)
	local parts = {}
	for piece in (arg .. '..'):gmatch('(.-)%.%.') do
		parts[#parts + 1] = literal(piece) or '…'
	end
	return table.concat(parts)
end

--- Removes Lua comments: long comments (`--[[ ]]`, `--[==[ ]==]`) first, then
--- line comments. A `--` inside a string literal is also cut, which can only
--- drop a dependency, never invent one.
--- @param src string
--- @return string
function p.stripComments(src)
	src = src:gsub('%-%-%[(=*)%[.-%]%1%]', '')
	return (src:gsub('%-%-[^\n]*', ''))
end

--- `Module:` plus the name with its first letter upper-cased, or nil when the
--- name is not in the Module namespace.
--- @param name string
--- @return string|nil
function p.normaliseModule(name)
	local rest = mw.text.trim(name):match('^[Mm][Oo][Dd][Uu][Ll][Ee]%s*:%s*(.+)$')
	if rest == nil then
		return nil
	end
	return 'Module:' .. rest:sub(1, 1):upper() .. rest:sub(2)
end

--- The module an `#invoke` names: with or without its `Module:` prefix.
--- @param target string
--- @return string|nil
function p.normaliseInvokeTarget(target)
	local name = mw.text.trim(target):gsub('^[Mm][Oo][Dd][Uu][Ll][Ee]%s*:%s*', '')
	if name == '' then
		return nil
	end
	return 'Module:' .. name:sub(1, 1):upper() .. name:sub(2)
end

--- The page a TemplateStyles `src` names. A `src` without a namespace is a
--- Template page, which is how TemplateStyles resolves it.
--- @param src string
--- @return string|nil
function p.normaliseStylesheet(src)
	local name = mw.text.trim(src)
	if name == '' then
		return nil
	end
	local namespace, rest = name:match('^([^:/]+):%s*(.+)$')
	if namespace == nil then
		namespace, rest = 'Template', name
	else
		namespace = mw.text.trim(namespace)
		local canonical = { module = 'Module', template = 'Template' }
		namespace = canonical[namespace:lower()] or namespace
	end
	return namespace .. ':' .. rest:sub(1, 1):upper() .. rest:sub(2)
end

--- Stylesheets named by `` tags.
--- @param text string|nil wikitext, or Lua source whose strings hold such tags
--- @return string[] sorted, unique
function p.styles(text)
	local found = {}
	for rest in stripHtmlComments(text or ''):gmatch(STYLES_TAG .. '([^>]*)') do
		local value = rest:match('^"([^"]*)"') or rest:match("^'([^']*)'") or rest:match('^([^%s>"\']+)')
		local name = value and p.normaliseStylesheet((value:gsub('/$', '')))
		if name then
			found[name] = true
		end
	end
	return sortedKeys(found)
end

--- A file's name as its page title shows it, without the namespace: underscores
--- as spaces, first letter upper-cased. Nil when `text` is not a file name with
--- an image extension. `%` is refused, so a Lua pattern such as `'%.svg'` is not
--- a file.
--- @param text string
--- @return string|nil
function p.normaliseFile(text)
	local name = mw.text.trim((text:gsub('[%s_]+', ' ')))
	name = name:gsub('^["\']', ''):gsub('["\']$', '')
	local base, extension = name:match('^(.-)%.(%w+)$')
	if
		base == nil
		or not FILE_EXTENSIONS[extension:lower()]
		or not base:find('%w')
		or base:sub(1, 1) == '.'
		or name:find('[#<>%[%]|{}/\\%%]')
	then
		return nil
	end
	return name:sub(1, 1):upper() .. name:sub(2)
end

--- @class DependenciesFileUrl
--- @field name string the file the URL names
--- @field path string the URL's md5 directories, e.g. `e/e8`

--- @class DependenciesFiles
--- @field names string[] file names without the namespace, sorted, unique
--- @field urls DependenciesFileUrl[] the media URLs among them, sorted by name then path
--- @field dynamic string[] names built while the page runs, such as `Sc-icon-brand-….svg`

--- @return table
local function collector()
	return { names = {}, urls = {}, dynamic = {} }
end

--- @param found table a collector()
--- @return DependenciesFiles
local function finish(found)
	local urls = {}
	for _, url in pairs(found.urls) do
		urls[#urls + 1] = url
	end
	table.sort(urls, function(a, b)
		return a.name .. '|' .. a.path < b.name .. '|' .. b.path
	end)
	return { names = sortedKeys(found.names), urls = urls, dynamic = sortedKeys(found.dynamic) }
end

--- Adds the files that media URLs in `text` name to `found`, and returns
--- `text` without those URLs.
--- @param text string
--- @param found table a collector()
--- @return string
local function scanUrls(text, found)
	for url in text:gmatch(MEDIA_URL) do
		local path = (url:match('//[^/]+/(.*)$'):gsub('^thumb/', ''))
		local first, pair, file = path:match('^(%x)/(%x%x)/([^/?#]+)')
		local name = first and file and p.normaliseFile(mw.uri.decode(file, 'PATH'))
		if name and pair:sub(1, 1) == first then
			found.names[name] = true
			found.urls[name .. '|' .. first .. '/' .. pair] = { name = name, path = first .. '/' .. pair }
		end
	end
	return (text:gsub(MEDIA_URL, ' '))
end

--- Adds the files `text` names to `found`: media URLs, and every segment
--- (see SEGMENT) that is a whole file name. A name holding `…` is built at
--- runtime and goes to `found.dynamic`.
--- @param text string
--- @param found table a collector()
--- @param transform (fun(segment: string): string)|nil applied to each segment first
local function scan(text, found, transform)
	for segment in scanUrls(text, found):gmatch(SEGMENT) do
		local name = p.normaliseFile(transform and transform(segment) or segment)
		if name and name:find('…', 1, true) then
			found.dynamic[name] = true
		elseif name then
			found.names[name] = true
		end
	end
end

--- Files a template names: `[[File:…]]`, a parameter value that is a whole
--- file name, and media URLs.
--- @param wikitext string|nil
--- @return DependenciesFiles
function p.files(wikitext)
	local found = collector()
	scan(stripHtmlComments(wikitext or ''), found)
	return finish(found)
end

--- Files a stylesheet names through media URLs.
--- @param css string|nil
--- @return DependenciesFiles
function p.stylesheetFiles(css)
	local found = collector()
	scanUrls(((css or ''):gsub('/%*.-%*/', '')), found)
	return finish(found)
end

--- @class DependenciesLiteral
--- @field value string
--- @field first integer the position of its opening quote or bracket
--- @field last integer the position of its closing quote or bracket

--- The string literals in Lua source, in order. Reads comments itself rather
--- than through stripComments(), which also cuts a `--` inside a string (as in
--- a CSS custom property) and so would leave a quote unmatched.
--- @param src string
--- @return DependenciesLiteral[]
local function literals(src)
	local found, i = {}, 1
	while true do
		local at = src:find('[%-\'"%[]', i)
		if at == nil then
			return found
		end
		local char = src:sub(at, at)
		if char == '-' then
			if src:sub(at + 1, at + 1) ~= '-' then
				i = at + 1
			else
				local level = src:match('^%[(=*)%[', at + 2)
				local stop
				if level then
					stop = select(2, src:find(']' .. level .. ']', at + 4 + #level, true))
				else
					stop = src:find('\n', at, true)
				end
				i = (stop or #src) + 1
			end
		elseif char == '[' then
			local level = src:match('^%[(=*)%[', at)
			if level == nil then
				i = at + 1
			else
				local open = at + #level + 2
				local close, stop = src:find(']' .. level .. ']', open, true)
				found[#found + 1] = { value = src:sub(open, (close or #src + 1) - 1), first = at, last = stop or #src }
				i = (stop or #src) + 1
			end
		else
			local parts, j = {}, at + 1
			while true do
				local special = src:find('[\\\n' .. char .. ']', j)
				if special == nil then
					parts[#parts + 1] = src:sub(j)
					j = #src
					break
				end
				parts[#parts + 1] = src:sub(j, special - 1)
				if src:sub(special, special) == '\\' then
					parts[#parts + 1] = src:sub(special + 1, special + 1)
					j = special + 2
				else
					j = special
					break
				end
			end
			found[#found + 1] = { value = table.concat(parts), first = at, last = j }
			i = j + 1
		end
	end
end

--- The text a concatenation ending in `list[k]` builds, with `…` for each part
--- that is not a literal; nil when `list[k]` is not concatenated onto anything.
--- @param src string
--- @param list DependenciesLiteral[]
--- @param k integer
--- @return string|nil
local function concatenated(src, list, k)
	local parts, i = { list[k].value }, k
	while true do
		local before = src:sub(i > 1 and list[i - 1].last + 1 or 1, list[i].first - 1)
		local head = before:match('^(.-)%s*%.%.%s*$')
		if head == nil then
			break
		end
		if head:find('%S') then
			table.insert(parts, 1, '…')
			-- `'a' .. name .. 'b'`: one name, call or index between two literals.
			local operand = (head:gsub('%b()', ''):gsub('%b[]', ''))
			if not operand:find('^%s*%.%.%s*[%w_%.:]+$') then
				break
			end
		end
		if i == 1 then
			break
		end
		i = i - 1
		table.insert(parts, 1, list[i].value)
	end
	if #parts == 1 then
		return nil
	end
	return table.concat(parts)
end

--- A segment with its `string.format` directives as `…`, unless it holds a
--- `%` that is not one, as a Lua pattern does.
--- @param segment string
--- @return string
local function formatted(segment)
	local replaced = segment:gsub(FORMAT_DIRECTIVE, '…')
	if replaced:find('%', 1, true) then
		return segment
	end
	return replaced
end

--- @param src string a Lua module's source
--- @return DependenciesFiles
local function luaFiles(src)
	local found = collector()
	local list = literals(src)
	for k, literal in ipairs(list) do
		local extension = literal.value:match('%.(%w+)%s*$')
		local joined = extension and FILE_EXTENSIONS[extension:lower()] and concatenated(src, list, k)
		scan(joined or literal.value, found, formatted)
	end
	return finish(found)
end

--- @class DependenciesLua
--- @field requires string[] wiki modules required, sorted, unique
--- @field loads string[] modules read with mw.loadData or mw.loadJsonData
--- @field dynamic string[] required patterns such as `Module:X/…`
--- @field strict boolean the module requires `strict`
--- @field styles string[] TemplateStyles pages loaded through frame:extensionTag or a tag in a string
--- @field files DependenciesFiles the files its strings name

--- @param src string|nil a Lua module's source
--- @return DependenciesLua
function p.lua(src)
	local files = luaFiles(src or '')
	src = p.stripComments(src or '')
	local found = { requires = {}, loads = {}, dynamic = {} }
	local strict = false

	local function add(list, arg)
		local value = literal(arg)
		if value == nil then
			local identifier = arg:match('^%s*([%a_][%w_]*)%s*$')
			if identifier then
				value = variableValue(src, identifier)
			end
		end
		if value == nil and arg:find('%.%.') then
			local joined = concatenation(arg)
			if joined:find('…', 1, true) then
				local pattern = p.normaliseModule(joined)
				-- `Module:…` (no literal text after the namespace) tells the reader
				-- nothing, so it is dropped rather than recorded.
				if pattern and pattern ~= 'Module:…' then
					found.dynamic[pattern] = true
				end
				return
			end
			value = joined
		end
		if value == nil then
			return
		end
		if value == 'strict' then
			strict = true
		end
		if BUILTINS[value] then
			return
		end
		local name = p.normaliseModule(value)
		if name then
			found[list][name] = true
		end
	end

	for _, call in ipairs(CALLS) do
		local head = '%f[%w_%.:]' .. call.pattern
		for arg in src:gmatch(head .. '%s*(%b())') do
			add(call.list, arg:sub(2, -2))
		end
		for quote, value in src:gmatch(head .. '%s*([\'"])(.-)%1') do
			add(call.list, quote .. value .. quote)
		end
		for arg in src:gmatch('%f[%w_]pcall%s*%(%s*' .. call.pattern .. '%s*,([^%),]+)') do
			add(call.list, arg)
		end
	end

	local styles = {}
	for _, sheet in ipairs(p.styles(src)) do
		styles[sheet] = true
	end
	for _, shape in ipairs({ '%b()', '%b{}' }) do
		for arg in src:gmatch('%f[%w_]extensionTag%s*(' .. shape .. ')') do
			if arg:lower():find('templatestyles', 1, true) then
				local _, value = arg:match('src%s*=%s*([\'"])(.-)%1')
				if value == nil then
					local identifier = arg:match('src%s*=%s*([%a_][%w_]*)%s*[,}]')
					value = identifier and variableValue(src, identifier)
				end
				local name = value and p.normaliseStylesheet(value)
				if name then
					styles[name] = true
				end
			end
		end
	end

	return {
		requires = sortedKeys(found.requires),
		loads = sortedKeys(found.loads),
		dynamic = sortedKeys(found.dynamic),
		strict = strict,
		styles = sortedKeys(styles),
		files = files,
	}
end

--- @class DependenciesInvoke
--- @field module string `Module:Name`
--- @field func string function name; a parameter such as `{{{1}}}` is kept verbatim

--- @param wikitext string|nil a template's source
--- @return DependenciesInvoke[] sorted by module then function, unique
function p.invokes(wikitext)
	local text = stripHtmlComments(wikitext or '')
	local seen, calls = {}, {}
	for target, func in text:gmatch('#[Ii][Nn][Vv][Oo][Kk][Ee]%s*:([^|{}]+)|%s*([^|}]*)') do
		local module = p.normaliseInvokeTarget(target)
		func = mw.text.trim(func)
		if func:sub(1, 3) == '{{{' then
			func = func .. '}}}'
		end
		local key = module and (module .. '|' .. func)
		if key and func ~= '' and not seen[key] then
			seen[key] = true
			calls[#calls + 1] = { module = module, func = func }
		end
	end
	table.sort(calls, function(a, b)
		return a.module .. '|' .. a.func < b.module .. '|' .. b.func
	end)
	return calls
end

return p