Star Citizen Wiki Difference between revisions of "Module:DPLlua"

Difference between revisions of "Module:DPLlua"

From the Star Citizen Wiki, the fidelity™ encyclopedia
m (1 revision imported)
(Sync with RS wiki)
 
Line 1: Line 1:
 
-- <nowiki>
 
-- <nowiki>
 
local dpl = {}
 
local dpl = {}
 +
local libraryUtil = require( 'libraryUtil' )
 +
local hasContent = require( 'Module:Paramtest' ).has_content
 +
local checkType = libraryUtil.checkType
 +
local checkTypeForNamedArg = libraryUtil.checkTypeForNamedArg
  
 
dpl.pipe = '¦'
 
dpl.pipe = '¦'
 
local dataContentMarker = '`#@@#`'
 
local dataContentMarker = '`#@@#`'
local usesInclude = {}
+
local allIncludedParamNames = {}
  
 
-- Custom function for splitting a string because mw.text.split() is waaay too slow
 
-- Custom function for splitting a string because mw.text.split() is waaay too slow
 
local function split( str, pattern, plain )
 
local function split( str, pattern, plain )
 
local res = {}
 
local res = {}
continue = true
+
local continue = true
 
local startIndex = 1
 
local startIndex = 1
  
Line 28: Line 32:
 
-- Also custom function for speed
 
-- Also custom function for speed
 
local function trim( str )
 
local function trim( str )
return string.match( str, '^%s*(.-)%s*$' )
+
return (string.gsub( str, '^%s+', '' ):gsub( '%s+$', '' ))
 +
end
 +
 
 +
local function mergeItem( tbl, key, item )
 +
if type( tbl[key] ) == 'table' and type( item ) == 'table' then
 +
for k in pairs( tbl[key] ) do
 +
mergeItem( tbl[key], k, item[k] )
 +
end
 +
elseif type( tbl[key] ) == 'table' then
 +
table.insert( tbl[key], item )
 +
else
 +
tbl[key] = { tbl[key], item }
 +
end
 
end
 
end
  
 
local escapeChars = {
 
local escapeChars = {
 
['{'] = '&#123;',
 
['{'] = '&#123;',
['\180'] = '&#123;', -- Wtf dpl...
 
 
['}'] = '&#125;',
 
['}'] = '&#125;',
['\181'] = '&#125;',
 
 
['['] = '&#91;',
 
['['] = '&#91;',
 
[']'] = '&#93;',
 
[']'] = '&#93;',
 
['|'] = '&#124;',
 
['|'] = '&#124;',
['-'] = '&#8208;',
+
['-'] = '&#8208;'
['\226'] = '',
 
['\157'] = '',
 
 
}
 
}
 
local function escape( str )
 
local function escape( str )
-- the \226\157\180\181 are used to match ❴ (U+E29DB4) and ❵ (U+E29DB5) wich are 3 bytes long so
+
return (string.gsub( str, '[{}%[%]|%-]', escapeChars ))
-- we can't use them directly inside [] patterns. Ustring would fix this but it's way too slow.
 
str = string.gsub( str, '[{}%[%]|%-\226\157\180\181]', escapeChars )
 
return str
 
 
end
 
end
  
Line 59: Line 68:
 
}
 
}
 
local function unEscape( str )
 
local function unEscape( str )
str = string.gsub( str, '&#%d+;', unEscapeChars )
+
return (string.gsub( str, '&#%d+;', unEscapeChars ))
return str
+
end
 +
 
 +
local function fixCurlyBrackets( str )
 +
-- the \226\157\180\181 are used to match ❴ (U+2774) and ❵ (U+2775) wich are 3 bytes long (UTF-8) so
 +
-- we can't use them directly inside [] patterns. Ustring would fix this but it's way too slow.
 +
return (string.gsub( str, '\226\157[\180\181]', { ['❴'] = '{', ['❵'] = '}' } ))
 
end
 
end
  
Line 91: Line 105:
 
'suppresserrors',
 
'suppresserrors',
 
'noresultsfooter',
 
'noresultsfooter',
'format'
+
'format',
 +
'groupMultiTemplateResults'
 
}
 
}
  
Line 100: Line 115:
  
 
local function formatInclude( query )
 
local function formatInclude( query )
 +
checkTypeForNamedArg( 'Module:DPLlua.ask', 'include', query, 'string' )
 
query = split( query, ',', true )
 
query = split( query, ',', true )
local count = #query
+
local includedParamNames = {}
 +
local sectionAttributes = {}
  
 
for i = 1, #query do
 
for i = 1, #query do
 
if query[i]:match( '%b{}' ) then -- Check if we are including a template
 
if query[i]:match( '%b{}' ) then -- Check if we are including a template
local templateName, params = query[i]:match( '{(.-)[¦|}]([^,]*)' )
+
local templateName, extra = query[i]:match( '{(.-)[¦|}](.*)' )
if params:find( '%S' ) then
+
if hasContent( extra ) then
params:gsub( '^:%-', '' )
+
local phantomTemplateName = extra:match( '^(.-)}' ) or extra:match( '^[./].+' )
query[i] = string.format( '{%s}%s', templateName, params )
+
local phantomTemplatePrefix = extra:match( '^(.-)}' ) and '' or templateName
for _ in params:gmatch( ':' ) do
+
local params = extra:gsub( '^.-}', '' ):gsub( '^[./].+', '' ):gsub( ':%-', '' )
count = count + 1
+
local sur = hasContent( phantomTemplateName ) and ('¦' .. phantomTemplatePrefix .. phantomTemplateName) or ''
 +
query[i] = string.format( '{%s%s}%s', templateName, sur, params )
 +
 
 +
if hasContent( phantomTemplateName ) then
 +
table.insert( includedParamNames, { name=phantomTemplatePrefix..phantomTemplateName, isTemplate=true, hasPhantomTemplate=true } )
 +
table.insert( sectionAttributes, { hasPhantomTemplate=true } )
 +
else
 +
for param in params:gmatch( ':([^:]*)' ) do
 +
param = trim( param )
 +
table.insert( includedParamNames, { name=templateName, isTemplate=true, param=param } )
 +
end
 +
table.insert( sectionAttributes, { hasPhantomTemplate=false } )
 
end
 
end
 
else
 
else
 
query[i] = string.format( '{%s¦DPLlua helper}', templateName ) -- Use a helper template to get all the parameters of our included template
 
query[i] = string.format( '{%s¦DPLlua helper}', templateName ) -- Use a helper template to get all the parameters of our included template
 +
table.insert( includedParamNames, { name=templateName, isTemplate=true, includeAll=true } )
 +
table.insert( sectionAttributes, { hasPhantomTemplate=false } )
 
end
 
end
 +
else
 +
table.insert( includedParamNames, { name=trim( query[i] ) } )
 +
table.insert( sectionAttributes, { hasPhantomTemplate=false } )
 
end
 
end
 
end
 
end
  
return table.concat( query, ',' ), count
+
return table.concat( query, ',' ), includedParamNames, sectionAttributes
 
end
 
end
  
Line 125: Line 158:
 
local count = query.count or 500
 
local count = query.count or 500
 
local offset = query.offset or 0
 
local offset = query.offset or 0
local _usesInclude = false
+
local usesInclude = false
 +
local includedParamNames = {}
 +
local sectionAttributes
 
query.count = nil
 
query.count = nil
 
query.offset = nil
 
query.offset = nil
  
-- We use table format when the include parameter is used to make sure we can
+
-- Use table format so we can place dataContentMarkers around each included parameter. The secseparator
-- differentiate between the results in case more than one item is included
+
-- is needed to add dataContentMarkers when a phantom template is used
 
local dplStringInclude =
 
local dplStringInclude =
 
[=[
 
[=[
 
{{#dpl:
 
{{#dpl:
|noresultsheader = @@
+
 
|count=%s
 
|count=%s
 
|offset=%s
 
|offset=%s
 
|%s
 
|%s
 
|table=,
 
|table=,
 +
|listseparators=,\n¦-\n¦[[%%PAGE%%¦]],,
 
|tablerow=%s
 
|tablerow=%s
 +
|secseparators=%s
 
}}]=]
 
}}]=]
  
Line 147: Line 184:
 
[=[
 
[=[
 
{{#dpl:
 
{{#dpl:
|noresultsheader = @@
+
 
|count=%s
 
|count=%s
 
|offset=%s
 
|offset=%s
Line 158: Line 195:
 
for i = 1, math.ceil( count / 500 ) do
 
for i = 1, math.ceil( count / 500 ) do
 
local params = {}
 
local params = {}
local includeCount = 0
 
  
 
for k, v in pairs( query ) do
 
for k, v in pairs( query ) do
 
if k == 'include' then
 
if k == 'include' then
v, includeCount = formatInclude( v )
+
v, includedParamNames, sectionAttributes = formatInclude( v )
_usesInclude =  true
+
usesInclude =  true
 +
end
 +
 
 +
if type( v ) == 'table' then
 +
for _, x in ipairs( v ) do
 +
table.insert( params, k .. '=' .. tostring( x ):gsub( '|', '¦' ) )
 +
end
 +
else
 +
table.insert( params, k .. '=' .. tostring( v ):gsub( '|', '¦' ) )
 
end
 
end
table.insert( params, k .. '=' .. tostring( v ):gsub( '|', '¦' ) )
 
 
end
 
end
  
if _usesInclude then
+
if usesInclude then
 +
local secseparators = ''
 +
for _, v in ipairs( sectionAttributes ) do
 +
if v.hasPhantomTemplate then
 +
-- Phantom templates need this because they ignore tablerow formatting
 +
secseparators = secseparators .. '¶¦' .. dataContentMarker .. ',' .. dataContentMarker .. ','
 +
else
 +
secseparators = secseparators .. '¶¦,,'
 +
end
 +
end
 +
 
 
table.insert( queries, string.format(
 
table.insert( queries, string.format(
 
dplStringInclude,
 
dplStringInclude,
Line 174: Line 227:
 
offset,
 
offset,
 
table.concat( params, '\n|' ),
 
table.concat( params, '\n|' ),
string.rep( dataContentMarker..'%%'..dataContentMarker..',', includeCount )
+
string.rep( dataContentMarker..'%%'..dataContentMarker..',', #includedParamNames ),
 +
secseparators
 
) )
 
) )
 
else
 
else
Line 189: Line 243:
 
end
 
end
  
table.insert( usesInclude, _usesInclude )
+
table.insert( allIncludedParamNames, includedParamNames )
 
 
 
return table.concat( queries )
 
return table.concat( queries )
 
end
 
end
  
local function toTable( query )
+
local function toTable( query, groupMultiTemplateResults )
local _usesInclude = table.remove( usesInclude, 1 )
+
local includedParamNames = table.remove( allIncludedParamNames, 1 )
 +
local usesInclude = #includedParamNames > 0
 
local res = {}
 
local res = {}
+
 
query = query:gsub( '<p>Extension:DynamicPageList .-</p>', function(item) res.error = item; return '' end)
+
query = query:gsub( '<p>Extension:DynamicPageList .-</p>', function(item) res.error = item; return '' end )
+
 
if not query:match( '^@@' ) then -- @@ is used when no result is found
+
if query:find( '^@@' ) then -- @@ is used when no result is found
if _usesInclude then
+
return res
query = query:gsub( '\127\'"`UNIQ%-%-nowiki%-%x+%-QINU`"\'\127', function(item) return '<nowiki>' .. mw.text.unstripNoWiki( item ) .. '</nowiki>' end )
+
end
query = query:gsub( dataContentMarker..'(.-)'..dataContentMarker, escape )
+
 
query = query:gsub( '{|.-|%-', '') -- Remove the header of the table
+
if usesInclude then
-- Replace the footer of the table width a row indicator. This effectively
+
query = query:gsub( dataContentMarker..'(.-)'..dataContentMarker, escape )
-- combines the output of multiple dpl queries when count > 500
+
end
query = query:gsub( '|}', '|-' )
+
 
end
+
query = trim( query )
+
query = split( query, '|-', true ) -- Results of the returned pages are separated by |-
query = trim( query )
+
 
query = split( query, '|-', true ) -- Results of the returned pages are separated by |-
+
for _, v in ipairs( query ) do
+
if hasContent( v ) and not v:find( '^@@' ) then
for _, v in ipairs( query ) do
+
v = trim( v )
if v:match( '%S' ) then
+
local title = v:match( '^|%[%[(.-)|' )
v = trim( v )
+
local rawDataList = v:match( '^|.-|.-|(.*)' ) -- This is everything after the title
local title = v:match( '^|%[%[(.-)|' )
+
 
local dataList = v:match( '^|.-|.-|(.*)' ) -- This is everything after the title
+
if not usesInclude then
+
if title and title ~= '' then
if not _usesInclude then
+
table.insert( res, title )
if title and title ~= '' then
+
end
table.insert( res, title )
+
else
end
+
-- When multiple includes are used (e.g. include={Template1},{Template2} or include={Template}:1:2) their results are separated by a pipe
else
+
rawDataList = split( rawDataList, '|', true )
-- When multiple includes are used (e.g. include={Template1},{Template2}) its results are separated by a pipe
+
local cleanedDataList = {}
dataList = split( dataList, '|', true )
+
 
local _dataList = {}
+
for _incIndex, dataItem in ipairs( rawDataList ) do
+
local incIndex = ((_incIndex - 1) % #includedParamNames) + 1 -- Needed in case the same template appears multiple times on the same page
for _, dataItem in ipairs( dataList ) do
+
dataItem = unEscape( dataItem )
dataItem = unEscape( dataItem )
+
dataItem = trim( dataItem )
 +
 
 +
if includedParamNames[ incIndex ].isTemplate and includedParamNames[ incIndex ].includeAll then -- Check if we included a full template
 
-- When we include an entire template we use the %ARGS% parameter supplied by dpl.
 
-- When we include an entire template we use the %ARGS% parameter supplied by dpl.
 
-- However all | characters are repaced with §, e.g.:
 
-- However all | characters are repaced with §, e.g.:
-- §nameLessParam
+
-- §namelessParam
 
-- §param = text [[wowee§link text]]
 
-- §param = text [[wowee§link text]]
 
-- §param2 = text {{something§something else}}
 
-- §param2 = text {{something§something else}}
 +
dataItem = dataItem:gsub( '\127\'"`UNIQ%-%-nowiki%-%x+%-QINU`"\'\127', function(item) return '<nowiki>' .. item .. '</nowiki>' end )
 +
dataItem = mw.text.unstripNoWiki( dataItem ) -- Unstrip nowiki so we can clean their content
 +
dataItem = fixCurlyBrackets( dataItem ) -- When using the %ARGS% dpl parameter, curly brackets are replaced with ❴ (U+2774) and ❵ (U+2775)
 
dataItem = dataItem:gsub( '%b{}', function(x) return x:gsub( '§', '|' ) end ) -- Restore pipe characters inside links and templates
 
dataItem = dataItem:gsub( '%b{}', function(x) return x:gsub( '§', '|' ) end ) -- Restore pipe characters inside links and templates
 
dataItem = dataItem:gsub( '%b[]', function(x) return x:gsub( '§', '|' ) end )
 
dataItem = dataItem:gsub( '%b[]', function(x) return x:gsub( '§', '|' ) end )
dataItem = trim( dataItem )
+
dataItem = dataItem:gsub( '<nowiki>(.-)</nowiki>', function(x) return mw.getCurrentFrame():extensionTag( 'nowiki', x ) end ) -- Restrip nowiki
+
local _dataItem = {}
if dataItem:match( '§' ) then -- Check if we included a template
+
 
dataItem = split( dataItem, '§', true )
+
if dataItem ~= '' then
local _dataItem = {}
+
dataItem = split( dataItem:sub( 3 ), '§' ) -- The sub(3) removes the first § at the start. § is 2 bytes wide so start at index 3
+
 
 
for i, item in ipairs( dataItem ) do
 
for i, item in ipairs( dataItem ) do
if i ~= 1 then -- skip first item as it is a false empty string created by splitting on § when the string started with a §
+
if item:find( '=' ) then -- Check if the parameter is named or unnamed
if item:find( '=' ) then -- Check if the parameter is named or unnamed
+
local param, value = item:match( '^%s*(.-)%s*=%s*(.-)%s*$' )
local param, value = item:match( '^%s*(.-)%s*=%s*(.-)%s*$' )
+
_dataItem[ param ] = value
_dataItem[ param ] = value
+
else
else
+
table.insert( _dataItem, trim( item ) )
table.insert( _dataItem, trim( item ) )
 
end
 
 
end
 
end
 
end
 
end
 
dataItem = _dataItem
 
 
end
 
end
+
 
table.insert( _dataList, dataItem )
+
dataItem = _dataItem
 +
end
 +
 
 +
local dataListIndex = groupMultiTemplateResults and 1 or math.ceil( _incIndex / #includedParamNames )
 +
if
 +
includedParamNames[ incIndex ].isTemplate and
 +
not includedParamNames[ incIndex ].includeAll and
 +
not includedParamNames[ incIndex ].hasPhantomTemplate
 +
then -- This means there was an include in the form 'include = {template}:param'
 +
local templateName = includedParamNames[ incIndex ].name
 +
local paramName = includedParamNames[ incIndex ].param
 +
paramName = tonumber( paramName ) or paramName -- Keep as string if tonumber fails
 +
cleanedDataList[ dataListIndex ] = cleanedDataList[ dataListIndex ] or {}
 +
cleanedDataList[ dataListIndex ][ templateName ] = cleanedDataList[ dataListIndex ][ templateName ] or {}
 +
 +
if groupMultiTemplateResults and _incIndex > #includedParamNames then
 +
mergeItem( cleanedDataList[ dataListIndex ][ templateName ], paramName, dataItem )
 +
else
 +
cleanedDataList[ dataListIndex ][ templateName ][ paramName ] = dataItem
 +
end
 +
else
 +
local templateName = includedParamNames[ incIndex ].name
 +
cleanedDataList[ dataListIndex ] = cleanedDataList[ dataListIndex ] or {}
 +
 +
if groupMultiTemplateResults and _incIndex > #includedParamNames then
 +
mergeItem( cleanedDataList[ dataListIndex ], templateName, dataItem )
 +
else
 +
cleanedDataList[ dataListIndex ][ templateName ] = dataItem
 +
end
 
end
 
end
+
end
if title and title ~= '' then
+
 
table.insert( res, { title=title, include=_dataList } )
+
if title and title ~= '' then
 +
for _, v in ipairs( cleanedDataList ) do
 +
table.insert( res, { title=title, include=v } )
 
end
 
end
 
end
 
end
Line 274: Line 359:
 
-- Combinig multiple dpl queries yields better performance than doing them sequentially
 
-- Combinig multiple dpl queries yields better performance than doing them sequentially
 
function dpl.ask( ... )
 
function dpl.ask( ... )
 +
local formatTime = os.clock()
 
local queries = { ... }
 
local queries = { ... }
 +
local wantsGrouping = {}
  
 
for i = 1, #queries do
 
for i = 1, #queries do
 +
checkType( 'Module:DPLlua.ask', i, queries[i], 'table' )
 +
table.insert( wantsGrouping, queries[i].groupMultiTemplateResults or false )
 
removeFormattingSettings( queries[i] )
 
removeFormattingSettings( queries[i] )
 
queries[i] = formatDpl( queries[i] )
 
queries[i] = formatDpl( queries[i] )
 
end
 
end
 +
formatTime = os.clock() - formatTime
  
 +
local DPLtime = os.clock()
 
queries = table.concat( queries, '[email protected]µ@$' )
 
queries = table.concat( queries, '[email protected]µ@$' )
local time = os.clock()
 
 
queries = mw.getCurrentFrame():preprocess( queries )
 
queries = mw.getCurrentFrame():preprocess( queries )
time = os.clock() - time
 
 
queries = split( queries, '[email protected]µ@$', true )
 
queries = split( queries, '[email protected]µ@$', true )
 +
DPLtime = os.clock() - DPLtime
  
 
for i = 1, #queries do
 
for i = 1, #queries do
queries[i] = toTable( queries[i] )
+
local parseTime = os.clock()
queries[i].time = time
+
queries[i] = toTable( queries[i], wantsGrouping[i] )
 +
parseTime = os.clock() - parseTime
 +
queries[i]['DPL time'] = DPLtime
 +
queries[i]['Parse time'] = math.floor( (formatTime + parseTime) * 1e5 ) / 1e5 -- os.clock() has a resolution of 10µs
 
end
 
end
  
 
return unpack( queries )
 
return unpack( queries )
 
end
 
end
 +
 +
-- function dpl.test()
 +
-- local time = os.clock()
 +
 +
-- local a, b = dpl.ask({
 +
--    namespace = 'Module',
 +
--    linksto = 'Module:Chart data',
 +
--    distinct = 'strict',
 +
--    ordermethod = 'title',
 +
--    nottitlematch = '%/doc¦%sandbox%¦Exchange/%¦Exchange historical/%¦Chart data',
 +
-- ignorecase = 'true',
 +
-- allowcachedresults = false
 +
-- },{
 +
--    namespace = 'Module',
 +
--    linksto = 'Module:Enum',
 +
--    distinct = 'strict',
 +
--    ordermethod = 'title',
 +
-- nottitlematch = '%/doc¦%sandbox%¦Exchange/%¦Exchange historical/%¦Enum',
 +
--    ignorecase = 'true',
 +
-- allowcachedresults = false
 +
-- })
 +
-- mw.logObject(a)
 +
-- mw.logObject(b)
 +
 +
-- local a, b = dpl.ask({
 +
--    namespace = 'Module',
 +
--    linksto = 'Module:Chart data',
 +
--    distinct = 'strict',
 +
--    ordermethod = 'title',
 +
--    nottitlematch = '%/doc¦%sandbox%¦Exchange/%¦Exchange historical/%¦Chart data',
 +
--    ignorecase = 'true',
 +
-- allowcachedresults = false
 +
-- },{
 +
-- namespace = '',
 +
-- ignorecase = 'true',
 +
-- uses = 'Template:Infobox Recipe',
 +
-- count = 50,
 +
-- include = '{Infobox Recipe},{Infobox Item}',
 +
-- allowcachedresults = false
 +
-- })
 +
-- mw.logObject(a)
 +
-- mw.logObject(b)
 +
 +
-- local a = dpl.ask{
 +
-- namespace = '',
 +
-- uses = 'Template:Infobox Recipe',
 +
-- include = '{Infobox Recipe}:skill:name,{Infobox Item}:update,{Infobox Item|test}',
 +
-- count = 50,
 +
-- ordermethod = 'title',
 +
-- }
 +
-- mw.logObject(a)
 +
 +
-- local q = dpl.ask{
 +
-- uses = "Template:Collections table",
 +
-- category = "Archaeology collections",
 +
-- -- include = "{Infobox collection}:reward,{Collections table}:1:2:3:4:5:6:7:8:9:10:11:12:13:14:15",
 +
-- include = "{Infobox collection}:reward,{Collections table}",
 +
-- count = 100
 +
-- }
 +
-- mw.logObject(q)
 +
 +
-- local q = dpl.ask{
 +
-- namespace = "",
 +
-- uses = "Template:Infobox spell",
 +
-- notcategory = {"Removed content", "Removed spells"},
 +
-- nottitlematch = {"Enchant Crossbow Bolt", "Storm of Armadyl"},
 +
-- include = "{Infobox spell}",
 +
-- }
 +
-- mw.logObject(q)
 +
 +
-- local list = dpl.ask{
 +
-- namespace = 'Template',
 +
-- uses = 'Template:Navbox',
 +
-- ordermethod = 'title',
 +
-- include = '{Navbox}:gtitle1:gtitle2',
 +
-- count = 1,
 +
-- offset = 3
 +
-- }
 +
-- mw.logObject(list)
 +
 +
-- local list = dpl.ask{
 +
-- namespace = 'User',
 +
-- titlematch = 'CephHunter/Sandbox/test1',
 +
-- include = '{User:CephHunter/Sandbox/test2|User:CephHunter/Sandbox/test3},{User:CephHunter/Sandbox/test3}:1',
 +
-- }
 +
-- mw.logObject(list)
 +
 +
-- mw.logObject(dpl.ask{
 +
-- namespace = 'User',
 +
-- ignorecase = 'true',
 +
-- titlematch = 'CephHunter/Sandbox/test1',
 +
-- include = '{User:CephHunter/Sandbox/test2}'
 +
-- })
 +
 +
-- mw.logObject(dpl.ask{
 +
-- namespace = 'Module',
 +
-- uses = 'Template:Helper module',
 +
-- titlematch = '%/doc',
 +
-- nottitlematch = 'Exchange/%|Exchange historical/%|Sandbox/%',
 +
-- ordermethod = 'title',
 +
-- include = '{Helper module}, {Helper module}:example',
 +
-- count = 1,
 +
-- offset = 13
 +
-- })
 +
 +
-- mw.logObject(dpl.ask{
 +
--        namespace = 'Module',
 +
--        titlematch = 'Chart data|Absorbative calculator',
 +
--        nottitlematch = 'Exchange/%|Exchange historical/%|Sandbox/%|%/doc|DPLlua%',
 +
--        ordermethod = 'title',
 +
--        include = '%0'
 +
-- })
 +
 +
-- mw.logObject(dpl.ask{
 +
--        uses = 'Template:Collections table',
 +
--        include = '{Collections table}',
 +
--        count = 5
 +
-- })
 +
 +
-- mw.log(os.clock()-time)
 +
-- end
  
 
return dpl
 
return dpl
 
-- </nowiki>
 
-- </nowiki>

Latest revision as of 04:14, 9 May 2021

Module documentation[view][edit][history][purge]
This documentation is transcluded from Module:DPLlua/doc. Changes can be proposed in the talk page.
Module:DPLlua requires Module:LibraryUtil.
Module:DPLlua requires Module:Paramtest.
Module:DPLlua is required by Module:DependencyList.

Module:DPLlua uses Template:DPLlua helper to make it possible to include all parameters of a template while maintaining good performance. See Module:DPLlua on RuneScape Wiki for more details.


-- <nowiki>
local dpl = {}
local libraryUtil = require( 'libraryUtil' )
local hasContent = require( 'Module:Paramtest' ).has_content
local checkType = libraryUtil.checkType
local checkTypeForNamedArg = libraryUtil.checkTypeForNamedArg

dpl.pipe = '¦'
local dataContentMarker = '`#@@#`'
local allIncludedParamNames = {}

-- Custom function for splitting a string because mw.text.split() is waaay too slow
local function split( str, pattern, plain )
	local res = {}
	local continue = true
	local startIndex = 1

	while continue do
		local i, j = string.find( str, pattern, startIndex, plain )
		if i then
			table.insert( res, string.sub( str, startIndex, i-1 ) )
			startIndex = j + 1
		else
			table.insert( res, string.sub( str, startIndex ) )
			continue = false
		end
	end

	return res
end

-- Also custom function for speed
local function trim( str )
	return (string.gsub( str, '^%s+', '' ):gsub( '%s+$', '' ))
end

local function mergeItem( tbl, key, item )
	if type( tbl[key] ) == 'table' and type( item ) == 'table' then
		for k in pairs( tbl[key] ) do
			mergeItem( tbl[key], k, item[k] )
		end
	elseif type( tbl[key] ) == 'table' then
		table.insert( tbl[key], item )
	else
		tbl[key] = { tbl[key], item }
	end
end

local escapeChars = {
	['{'] = '&#123;',
	['}'] = '&#125;',
	['['] = '&#91;',
	[']'] = '&#93;',
	['|'] = '&#124;',
	['-'] = '&#8208;'
}
local function escape( str )
	return (string.gsub( str, '[{}%[%]|%-]', escapeChars ))
end

local unEscapeChars = {
	['&#123;'] = '{',
	['&#125;'] = '}',
	['&#91;'] = '[',
	['&#93;'] = ']',
	['&#124;'] = '|',
	['&#8208;'] = '-'
}
local function unEscape( str )
	return (string.gsub( str, '&#%d+;', unEscapeChars ))
end

local function fixCurlyBrackets( str )
	-- the \226\157\180\181 are used to match ❴ (U+2774) and ❵ (U+2775) wich are 3 bytes long (UTF-8) so
	-- we can't use them directly inside [] patterns. Ustring would fix this but it's way too slow.
	return (string.gsub( str, '\226\157[\180\181]', { ['❴'] = '{', ['❵'] = '}' } ))
end

local function removeFormattingSettings( query )
	local toRemove = {
		'mode',
		'table',
		'tablerow',
		'tablesortcol',
		'headingmode',
		'headingcount',
		'listattr',
		'itemattr',
		'hlistattr',
		'hitemattr',
		'userdateformat',
		'shownamespace',
		'escapelinks',
		'titlemaxlength',
		'replaceintitle',
		'columns',
		'rows',
		'rowsize',
		'rowcolformat',
		'resultsheader',
		'resultsfooter',
		'oneresultheader',
		'oneresultfooter',
		'noresultsheader',
		'suppresserrors',
		'noresultsfooter',
		'format',
		'groupMultiTemplateResults'
	}

	for _, k in ipairs( toRemove ) do
		query[k] = nil
	end
end

local function formatInclude( query )
	checkTypeForNamedArg( 'Module:DPLlua.ask', 'include', query, 'string' )
	query = split( query, ',', true )
	local includedParamNames = {}
	local sectionAttributes = {}

	for i = 1, #query do
		if query[i]:match( '%b{}' ) then -- Check if we are including a template
			local templateName, extra = query[i]:match( '{(.-)[¦|}](.*)' )
			if hasContent( extra ) then
				local phantomTemplateName = extra:match( '^(.-)}' ) or extra:match( '^[./].+' )
				local phantomTemplatePrefix = extra:match( '^(.-)}' ) and '' or templateName
				local params = extra:gsub( '^.-}', '' ):gsub( '^[./].+', '' ):gsub( ':%-', '' )
				local sur = hasContent( phantomTemplateName ) and ('¦' .. phantomTemplatePrefix .. phantomTemplateName) or ''
				query[i] = string.format( '{%s%s}%s', templateName, sur, params )

				if hasContent( phantomTemplateName ) then
					table.insert( includedParamNames, { name=phantomTemplatePrefix..phantomTemplateName, isTemplate=true, hasPhantomTemplate=true } )
					table.insert( sectionAttributes, { hasPhantomTemplate=true } )
				else
					for param in params:gmatch( ':([^:]*)' ) do
						param = trim( param )
						table.insert( includedParamNames, { name=templateName, isTemplate=true, param=param } )
					end
					table.insert( sectionAttributes, { hasPhantomTemplate=false } )
				end
			else
				query[i] = string.format( '{%s¦DPLlua helper}', templateName ) -- Use a helper template to get all the parameters of our included template
				table.insert( includedParamNames, { name=templateName, isTemplate=true, includeAll=true } )
				table.insert( sectionAttributes, { hasPhantomTemplate=false } )
			end
		else
			table.insert( includedParamNames, { name=trim( query[i] ) } )
			table.insert( sectionAttributes, { hasPhantomTemplate=false } )
		end
	end

	return table.concat( query, ',' ), includedParamNames, sectionAttributes
end

local function formatDpl( query )
	local queries = {}
	local count = query.count or 500
	local offset = query.offset or 0
	local usesInclude = false
	local includedParamNames = {}
	local sectionAttributes
	query.count = nil
	query.offset = nil

	-- Use table format so we can place dataContentMarkers around each included parameter. The secseparator
	-- is needed to add dataContentMarkers when a phantom template is used
	local dplStringInclude =
[=[
{{#dpl:
|[email protected]@
|count=%s
|offset=%s
|%s
|table=,
|listseparators=,\n¦-\n¦[[%%PAGE%%¦]],,
|tablerow=%s
|secseparators=%s
}}]=]

	-- Table format requires an include statement so we use format instead.
	-- This is also a lot faster than adding an empty include statement
	local dplStringNoInclude =
[=[
{{#dpl:
|[email protected]@
|count=%s
|offset=%s
|%s
|format=,¦-¦[[%%PAGE%%¦]],,
}}]=]

	-- Auto generate more than one dpl if count > 500
	-- The results of these are later combined
	for i = 1, math.ceil( count / 500 ) do
		local params = {}

		for k, v in pairs( query ) do
			if k == 'include' then
				v, includedParamNames, sectionAttributes = formatInclude( v )
				usesInclude =  true
			end

			if type( v ) == 'table' then
				for _, x in ipairs( v ) do
					table.insert( params, k .. '=' .. tostring( x ):gsub( '|', '¦' ) )
				end
			else
				table.insert( params, k .. '=' .. tostring( v ):gsub( '|', '¦' ) )
			end
		end

		if usesInclude then
			local secseparators = ''
			for _, v in ipairs( sectionAttributes ) do
				if v.hasPhantomTemplate then
					-- Phantom templates need this because they ignore tablerow formatting
					secseparators = secseparators .. '¶¦' .. dataContentMarker .. ',' .. dataContentMarker .. ','
				else
					secseparators = secseparators .. '¶¦,,'
				end
			end

			table.insert( queries, string.format(
				dplStringInclude,
				count > 500 and 500 or count,
				offset,
				table.concat( params, '\n|' ),
				string.rep( dataContentMarker..'%%'..dataContentMarker..',', #includedParamNames ),
				secseparators
			) )
		else
			table.insert( queries, string.format(
				dplStringNoInclude,
				count > 500 and 500 or count,
				offset,
				table.concat( params, '\n|' )
			) )
		end

		count = count - 500
		offset = offset + 500
	end

	table.insert( allIncludedParamNames, includedParamNames )
	return table.concat( queries )
end

local function toTable( query, groupMultiTemplateResults )
	local includedParamNames = table.remove( allIncludedParamNames, 1 )
	local usesInclude = #includedParamNames > 0
	local res = {}

	query = query:gsub( '<p>Extension:DynamicPageList .-</p>', function(item) res.error = item; return '' end )

	if query:find( '^@@' ) then -- @@ is used when no result is found
		return res
	end

	if usesInclude then
		query = query:gsub( dataContentMarker..'(.-)'..dataContentMarker, escape )
	end

	query = trim( query )
	query = split( query, '|-', true ) -- Results of the returned pages are separated by |-

	for _, v in ipairs( query ) do
		if hasContent( v ) and not v:find( '^@@' ) then
			v = trim( v )
			local title = v:match( '^|%[%[(.-)|' )
			local rawDataList = v:match( '^|.-|.-|(.*)' ) -- This is everything after the title

			if not usesInclude then
				if title and title ~= '' then
					table.insert( res, title )
				end
			else
				-- When multiple includes are used (e.g. include={Template1},{Template2} or include={Template}:1:2) their results are separated by a pipe
				rawDataList = split( rawDataList, '|', true )
				local cleanedDataList = {}

				for _incIndex, dataItem in ipairs( rawDataList ) do
					local incIndex = ((_incIndex - 1) % #includedParamNames) + 1 -- Needed in case the same template appears multiple times on the same page
					dataItem = unEscape( dataItem )
					dataItem = trim( dataItem )

					if includedParamNames[ incIndex ].isTemplate and includedParamNames[ incIndex ].includeAll then -- Check if we included a full template
						-- When we include an entire template we use the %ARGS% parameter supplied by dpl.
						-- However all | characters are repaced with §, e.g.:
						-- §namelessParam
						-- §param = text [[wowee§link text]]
						-- §param2 = text {{something§something else}}
						dataItem = dataItem:gsub( '\127\'"`UNIQ%-%-nowiki%-%x+%-QINU`"\'\127', function(item) return '<nowiki>' .. item .. '</nowiki>' end )
						dataItem = mw.text.unstripNoWiki( dataItem ) -- Unstrip nowiki so we can clean their content
						dataItem = fixCurlyBrackets( dataItem ) -- When using the %ARGS% dpl parameter, curly brackets are replaced with ❴ (U+2774) and ❵ (U+2775)
						dataItem = dataItem:gsub( '%b{}', function(x) return x:gsub( '§', '|' ) end ) -- Restore pipe characters inside links and templates
						dataItem = dataItem:gsub( '%b[]', function(x) return x:gsub( '§', '|' ) end )
						dataItem = dataItem:gsub( '<nowiki>(.-)</nowiki>', function(x) return mw.getCurrentFrame():extensionTag( 'nowiki', x ) end ) -- Restrip nowiki
						local _dataItem = {}

						if dataItem ~= '' then
							dataItem = split( dataItem:sub( 3 ), '§' ) -- The sub(3) removes the first § at the start. § is 2 bytes wide so start at index 3

							for i, item in ipairs( dataItem ) do
								if item:find( '=' ) then -- Check if the parameter is named or unnamed
									local param, value = item:match( '^%s*(.-)%s*=%s*(.-)%s*$' )
									_dataItem[ param ] = value
								else
									table.insert( _dataItem, trim( item ) )
								end
							end
						end

						dataItem = _dataItem
					end

					local dataListIndex = groupMultiTemplateResults and 1 or math.ceil( _incIndex / #includedParamNames )
					if
						includedParamNames[ incIndex ].isTemplate and
						not includedParamNames[ incIndex ].includeAll and
						not includedParamNames[ incIndex ].hasPhantomTemplate
					then -- This means there was an include in the form 'include = {template}:param'
						local templateName = includedParamNames[ incIndex ].name
						local paramName = includedParamNames[ incIndex ].param
						paramName = tonumber( paramName ) or paramName -- Keep as string if tonumber fails
						cleanedDataList[ dataListIndex ] = cleanedDataList[ dataListIndex ] or {}
						cleanedDataList[ dataListIndex ][ templateName ] = cleanedDataList[ dataListIndex ][ templateName ] or {}
						
						if groupMultiTemplateResults and _incIndex > #includedParamNames then
							mergeItem( cleanedDataList[ dataListIndex ][ templateName ], paramName, dataItem )
						else
							cleanedDataList[ dataListIndex ][ templateName ][ paramName ] = dataItem
						end
					else
						local templateName = includedParamNames[ incIndex ].name
						cleanedDataList[ dataListIndex ] = cleanedDataList[ dataListIndex ] or {}
						
						if groupMultiTemplateResults and _incIndex > #includedParamNames then
							mergeItem( cleanedDataList[ dataListIndex ], templateName, dataItem )
						else
							cleanedDataList[ dataListIndex ][ templateName ] = dataItem
						end
					end
				end

				if title and title ~= '' then
					for _, v in ipairs( cleanedDataList ) do
						table.insert( res, { title=title, include=v } )
					end
				end
			end
		end
	end

	return res
end

-- Accepts a series of tables each containig the settings for a dpl query.
-- Combinig multiple dpl queries yields better performance than doing them sequentially
function dpl.ask( ... )
	local formatTime = os.clock()
	local queries = { ... }
	local wantsGrouping = {}

	for i = 1, #queries do
		checkType( 'Module:DPLlua.ask', i, queries[i], 'table' )
		table.insert( wantsGrouping, queries[i].groupMultiTemplateResults or false )
		removeFormattingSettings( queries[i] )
		queries[i] = formatDpl( queries[i] )
	end
	formatTime = os.clock() - formatTime

	local DPLtime = os.clock()
	queries = table.concat( queries, '[email protected]µ@$' )
	queries = mw.getCurrentFrame():preprocess( queries )
	queries = split( queries, '[email protected]µ@$', true )
	DPLtime = os.clock() - DPLtime

	for i = 1, #queries do
		local parseTime = os.clock()
		queries[i] = toTable( queries[i], wantsGrouping[i] )
		parseTime = os.clock() - parseTime
		queries[i]['DPL time'] = DPLtime
		queries[i]['Parse time'] = math.floor( (formatTime + parseTime) * 1e5 ) / 1e5 -- os.clock() has a resolution of 10µs
	end

	return unpack( queries )
end

-- function dpl.test()
-- 	local time = os.clock()

	-- local a, b = dpl.ask({
	--     namespace = 'Module',
	--     linksto = 'Module:Chart data',
	--     distinct = 'strict',
	--     ordermethod = 'title',
	--     nottitlematch = '%/doc¦%sandbox%¦Exchange/%¦Exchange historical/%¦Chart data',
	-- 	ignorecase = 'true',
	-- 	allowcachedresults = false
	-- },{
	--     namespace = 'Module',
	--     linksto = 'Module:Enum',
	--     distinct = 'strict',
	--     ordermethod = 'title',
	-- 	nottitlematch = '%/doc¦%sandbox%¦Exchange/%¦Exchange historical/%¦Enum',
	--     ignorecase = 'true',
	-- 	allowcachedresults = false
	-- })
	-- mw.logObject(a)
	-- mw.logObject(b)

	-- local a, b = dpl.ask({
	--     namespace = 'Module',
	--     linksto = 'Module:Chart data',
	--     distinct = 'strict',
	--     ordermethod = 'title',
	--     nottitlematch = '%/doc¦%sandbox%¦Exchange/%¦Exchange historical/%¦Chart data',
	--     ignorecase = 'true',
	-- 	allowcachedresults = false
	-- },{
	-- 	namespace = '',
	-- 	ignorecase = 'true',
	-- 	uses = 'Template:Infobox Recipe',
	-- 	count = 50,
	-- 	include = '{Infobox Recipe},{Infobox Item}',
	-- 	allowcachedresults = false
	-- })
	-- mw.logObject(a)
	-- mw.logObject(b)

	-- local a = dpl.ask{
	-- 	namespace = '',
	-- 	uses = 'Template:Infobox Recipe',
	-- 	include = '{Infobox Recipe}:skill:name,{Infobox Item}:update,{Infobox Item|test}',
	-- 	count = 50,
	-- 	ordermethod = 'title',
	-- }
	-- mw.logObject(a)

	-- local q = dpl.ask{
	-- 	uses = "Template:Collections table",
	-- 	category = "Archaeology collections",
	-- 	-- include = "{Infobox collection}:reward,{Collections table}:1:2:3:4:5:6:7:8:9:10:11:12:13:14:15",
	-- 	include = "{Infobox collection}:reward,{Collections table}",
	-- 	count = 100
	-- }
	-- mw.logObject(q)
	
	-- local q = dpl.ask{
	-- 	namespace = "",
	-- 	uses = "Template:Infobox spell",
	-- 	notcategory = {"Removed content", "Removed spells"},
	-- 	nottitlematch = {"Enchant Crossbow Bolt", "Storm of Armadyl"},
	-- 	include = "{Infobox spell}",
	-- }
	-- mw.logObject(q)

-- 	local list = dpl.ask{
-- 		namespace = 'Template',
-- 		uses = 'Template:Navbox',
-- 		ordermethod = 'title',
-- 		include = '{Navbox}:gtitle1:gtitle2',
-- 		count = 1,
-- 		offset = 3
-- 	}
-- 	mw.logObject(list)

-- 	local list = dpl.ask{
-- 		namespace = 'User',
-- 		titlematch = 'CephHunter/Sandbox/test1',
-- 		include = '{User:CephHunter/Sandbox/test2|User:CephHunter/Sandbox/test3},{User:CephHunter/Sandbox/test3}:1',
-- 	}
-- 	mw.logObject(list)

-- 	mw.logObject(dpl.ask{
-- 		namespace = 'User',
-- 		ignorecase = 'true',
-- 		titlematch = 'CephHunter/Sandbox/test1',
-- 		include = '{User:CephHunter/Sandbox/test2}'
-- 	})

-- 	mw.logObject(dpl.ask{
-- 		namespace = 'Module',
-- 		uses = 'Template:Helper module',
-- 		titlematch = '%/doc',
-- 		nottitlematch = 'Exchange/%|Exchange historical/%|Sandbox/%',
-- 		ordermethod = 'title',
-- 		include = '{Helper module}, {Helper module}:example',
-- 		count = 1,
-- 		offset = 13
-- 	})

-- 	mw.logObject(dpl.ask{
--         namespace = 'Module',
--         titlematch = 'Chart data|Absorbative calculator',
--         nottitlematch = 'Exchange/%|Exchange historical/%|Sandbox/%|%/doc|DPLlua%',
--         ordermethod = 'title',
--         include = '%0'
-- 	})

-- 	mw.logObject(dpl.ask{
--         uses = 'Template:Collections table',
--         include = '{Collections table}',
--         count = 5
-- 	})

-- 	mw.log(os.clock()-time)
-- end

return dpl
-- </nowiki>
Star Citizen Wiki uses cookies to keep session information and analytics to provide you a better experience.