View source for Module:Creator

--[[  
  __  __           _       _         ____                _             
 |  \/  | ___   __| |_   _| | ___ _ / ___|_ __ ___  __ _| |_ ___  _ __ 
 | |\/| |/ _ \ / _` | | | | |/ _ (_) |   | '__/ _ \/ _` | __/ _ \| '__|
 | |  | | (_) | (_| | |_| | |  __/_| |___| | |  __/ (_| | || (_) | |   
 |_|  |_|\___/ \__,_|\__,_|_|\___(_)\____|_|  \___|\__,_|\__\___/|_|   

This module is intended to be the engine behind "Template:Creator".

Please do not modify this code without applying the changes first at 
"Module:Creator/sandbox" and testing at "Module:Creator/testcases".

Authors and maintainers:
* User:Jarekt - original version 

Handling of the fields
 ==============================================================================
 |field name        | property  | pull | push | missing  | mismatch | redundant 
 ==============================================================================
 |Name              | label     | 1    |  0   |          |          |
 |Alternative names | aliases   | 1    |  0   |          |          |
 |                  |P742,P1477,|      |      |          |          |
 |                  |P1782,P1787|      |      |          |          |
 |Sortkey           | P734,P735 | 1    |  0   |          |          | 1
 |Birthdate         | P569      | 1    |  1   |  1       | 1        | 1 
 |Deathdate         | P570      | 1    |  1   |  1       | 1        | 1 
 |Birthloc          | P19       | 1    |  1   |  1       | 1        | 1
 |Deathloc          | P20       | 1    |  1   |  1       | 1        | 1
 |Workperiod        |P2031,P2032| 1    |      |          |          |
 |                  | P1317     |      |      |          |          |
 |Workloc           | P937      | 1    |  1   |  1       | 1        | 1
 |Image             | P18       | 1    |  1   |  1       | 1        | 1
 |Homecat           | P373      | 1    |  1   |  1       | 1        | 1
 |Nationality       | P27, P172 | 1    |      |  1       | 1        | 1
 |Gender            | P21       | 1    |  1   |  1       | 1        | 1
 |Occupation        | P106      | 1    |      |  1       |          | 1
 |Linkback          | P1472     | 1    |  1   |  1       | 1        | 1
 |Wikisource        | sitelinks | 1    |  0   |          | 0        | 1
 |Wikiquote         | sitelinks | 1    |  0   |          | 0        | 1
 ===============================================================================
 
 pull - can we pull data from wikidata ? 
   - 1  - commons then wikidata
   -    - not implemented yet
   - 0  - will not implement
 push      - upload to wikidata through quick statements?
 missing   - detect if missing on Wikidata
 mismatch  - detect mismatch between wikidata and commons
 redundant - detect if redundant identical values on wikidata and commons
]]
local getLabel            = require("Module:Wikidata label")._getLabel            -- used for creation of name based on wikidata
local Wikidata_label      = require("Module:Wikidata label")                      -- used for creation of name based on wikidata
local getDate             = require("Module:Wikidata date")._date                 -- used for processing of date properties
local qualifierDate       = require("Module:Wikidata date")._qualifierDate        -- used for processing of date qualifiers
local authorityControl    = require("Module:Authority control")._authorityControl -- used for formatting of Authority control row
local alterName           = require("Module:Name")._name                          -- used for adding "option" fields to "name"
local City                = require("Module:City")                                -- used to add wikidata bases links to names of places
local ISOdate             = require("Module:ISOdate")                             -- used for internationalization of dates
local NationAndOccupation = require("Module:NationAndOccupation")._NationAndOccupation
local valid_date          = require("Module:Calendar")._valid_date
local labels              = require("Module:I18n/creator")
local TagQS               = require('Module:TagQS')
local core                = require("Module:core")
local messageBox          = require("Module:Message box")

-- ==================================================
-- === Internal functions ===========================
-- ==================================================

local function empty2nil(str)
	if str=='' then
		return nil
	else 
		return str;
	end
end

local function intersect(A, B) -- find intersection of tables A and B
	local ret = {}
	for _, a in ipairs(A or {}) do
		for _, b in ipairs(B or {}) do
			if a==b then 
				table.insert(ret, b) 
			end
		end
	end
	return ret
end

local function isodate2timestamp(dateStr)
-- convert isodate to timestamp used by quick statements
	local tStamp = nil
	if string.match(dateStr,"^%d%d%d%d$") then               -- if YYYY  format 
		tStamp = '+' .. dateStr .. '-00-00T00:00:00Z/9'
	elseif string.match(dateStr,"^%d%d%d%d%-%d%d$") then      -- if YYYY-MM format 
		tStamp = '+' .. dateStr .. '-00T00:00:00Z/10'
	elseif string.match(dateStr,"^%d%d%d%d%-%d%d%-%d%d$") then  -- if YYYY-MM-DD format 
		tStamp = '+' .. dateStr .. 'T00:00:00Z/11'
	end
	return tStamp
end

local function filepage_warningbox(text, lang, qCode)
	local boxArgs = {}
	
	boxArgs.type = 'content'
	boxArgs.text = string.format(core.langSwitch(labels[text],lang), qCode)
	
	return messageBox.main("mbox", boxArgs);
end
-- ====================================================================
-- This function is responsible for producing HTML of a single row of the template
-- At this stage all the fields are already filed. There is either one or two fields
-- INPUTS:
-- * param1 and param2 - structures for 2 fields containing fields:
--    - tag      - I18n tag used for localization of the field name. Usually name of page in MediaWiki namespace which was imported from translatewiki.org. 
--                 Alternative is to pass already translated field name.
--    - field    - field content
--    - id       - ID tag added to HTML's <td> cell. if IDs of 2 fields ar the same than we ignore the second one
--    - wrapper  - some fields need a <span class=...> wrapper around the field content 
-- ====================================================================
local function Build_html_row(param1, param2, args)
	local tag, headerCell, cell2, cell3
	local field1 = args[param1.field]
	local field2 = args[param2.field]
	if field1=='' then field1=nil; end
	if field2=='' then field2=nil; end
	if not (field1 or field2 or args.demo) then 
		return nil
	end
	if field2 then tag = param2.tag	else tag = param1.tag	end -- use different tag based on presence of field2
	if string.sub(tag,1,10) == 'wm-license' then
		tag = mw.message.new( tag ):inLanguage(args.lang):plain() -- label message in args.lang language
	end
	headerCell = string.format('<th scope="row">%s</th>\n', tag)
	if param1.id==param2.id then -- 2 cell row
		cell2 = string.format('<td colspan="2" class="fullwidth" id="%s">'.. param1.wrapper ..'</td>', param1.id, field1 or '')
		cell3 = ''
	else                         -- 3 cell row
		cell2 = string.format('<td class="halfwidth" id="%s">\n%s</td>', param1.id, field1 or '')
		cell3 = string.format('<td class="halfwidth" id="%s">\n%s</td>', param2.id, field2 or '')
	end
	return string.format('<tr>\n%s%s%s</tr>\n', headerCell, cell2, cell3)
end

-- ====================================================================
-- === This function is just responsible for producing HTML of the  ===
-- === template. At this stage all the fields are already filed     ===
-- ====================================================================
local function Build_html(args, cats)
	local field

	-- Top line with Creator name, lifespan and link icons -
	field = string.format('<span class="fn" id="creator"><bdi>%s\n</bdi></span>&nbsp;%s', args.name or 'missing name', args.lifespan or '') 
	if args.linkback then 
	    field = string.format('%s&nbsp;[[File:Blue pencil.svg|15px|link=Creator:%s]]', field, args.linkback) 
	end
	if args.wikidata then -- Wikidata Link
		field = string.format('%s&nbsp;[[File:Wikidata-logo.svg|20px|wikidata:%s|link=wikidata:%s]]', field, args.wikidata, args.wikidata) 
	end
	if args.wikisource then --Wikisource link
		field = string.format('%s&nbsp;[[File:Wikisource-logo.svg|15px|%s|link=%s]]', field, args.wikisource, args.wikisource) 
	end
	if args.wikiquote then --Wikiquote link
		field = string.format('%s&nbsp;[[File:Wikiquote-logo.svg|15px|%s|link=%s]]', field, args.wikiquote, args.wikiquote) 
	end
	if args.QS then -- quick_statement link to upload missing info to wikidata
		field = string.format('%s&nbsp;%s', field, args.QS) 
	end

	-- Provide our own collapsible toggle in the th, which is image based
	-- This avoids depending on English for anonymous users, but it's a bit of a hack.
	local arrowtoggle = string.format(
		' <div class="mw-collapsible-toggle %s"><span class="mw-collapsible-arrowtoggle">&nbsp;</span></div>',
		(args.collapse or args.namespace == 6) and 'mw-collapsible-toggle-collapsed' or 'mw-collapsible-toggle-expanded'
	);
	local line = string.format('<th colspan="4">%s%s</th>', field, arrowtoggle) 
	local results = {}
	table.insert(results, string.format('<tr>\n%s\n</tr>\n', line))
	
	-- add other fields
	local param = {
		{tag='wm-license-creator-alternative-names'          , field='alternative_names', id='fileinfotpl_creator_alt-name_value'   , wrapper='<div class="nickname">\n%s</div>' },
		{tag='wm-license-creator-description'                , field='description'      , id='fileinfotpl_creator_desc_value'       , wrapper='%s' },
		{tag='wm-license-creator-date-of-birth'              , field='birthdate'        , id='fileinfotpl_creator_birthdate_value'  , wrapper='%s' },
		{tag='wm-license-creator-date-of-birth-and-death'    , field='deathdate'        , id='fileinfotpl_creator_deathdate_value'  , wrapper='%s' },
		{tag='wm-license-creator-location-of-birth'          , field='birthloc'         , id='fileinfotpl_creator_birthloc_value'   , wrapper='%s' },
		{tag='wm-license-creator-location-of-birth-and-death', field='deathloc'         , id='fileinfotpl_creator_deathloc_value'   , wrapper='%s' },
		{tag='wm-license-creator-work-period'                , field='workperiod'       , id='fileinfotpl_creator_work-period_value', wrapper='%s' },
		{tag='wm-license-creator-work-location'              , field='workloc'          , id='fileinfotpl_creator_work-location'    , wrapper='<div class="locality">\n%s</div>' },
		{tag=args.authority_tag                              , field='authority'        , id='fileinfotpl_creator_authority_value'  , wrapper='%s' },
		{tag='wm-license-artwork-references'                 , field='references'       , id='fileinfotpl_creator_references'       , wrapper='<div>\n%s</div>'}
	}
	table.insert(results, Build_html_row(param[ 1], param[ 1], args))
	table.insert(results, Build_html_row(param[ 2], param[ 2], args))
	table.insert(results, Build_html_row(param[ 3], param[ 4], args))
	table.insert(results, Build_html_row(param[ 5], param[ 6], args))
	table.insert(results, Build_html_row(param[ 7], param[ 7], args))
	table.insert(results, Build_html_row(param[ 8], param[ 8], args))
	table.insert(results, Build_html_row(param[ 9], param[ 9], args))
	table.insert(results, Build_html_row(param[10], param[10], args))
	
	-- Image on the Left
	if not args.image and args.demo then
		args.image = 'Silver - replace this image male.svg'
	end
	if args.image then --Wikiquote link
		field = string.format('[[File:%s|120x360px|alt=%s|class=photo]]', args.image, args.name or '') 
	    local n = #results -- number of rows below 
		line  = string.format('<td rowspan="%i" style="width:120px" id="fileinfotpl_creator_image"><span class="wpImageAnnotatorControl wpImageAnnotatorOff">%s</span></td>', n, field) 
		table.insert(results, 2, string.format('<tr>\n%s\n</tr>\n', line)	)
	end
	results = table.concat(results)
	
	-- Template styles
	-- We should make this sandbox aware
	local templatestyles = mw.getCurrentFrame():extensionTag{
		name = 'templatestyles', args = { src = 'Module:Creator/styles.css' }
	}

	-- build table
	local dir = mw.language.new( args.lang ):getDir()
	local text_align = ((dir=='ltr') and 'left') or 'right'
	local collapsed = ''
	if args.collapse or args.namespace == 6 then
		collapsed = 'mw-collapsed'
	end
	local style = string.format('class="commons-creator-table mw-collapsible %s mw-content-%s" dir="%s" lang="%s"',
		collapsed, dir, dir, args.lang)
	results = string.format('<table %s>\n%s\n</table>\n', style, results)
	results = string.format('<div class="commons-creator vcard">\n%s\n%s\n</div>\n', templatestyles, results)
	
	-- add references and documentation which are only visible in creator namespace
	if args.namespace==100 or args.demo then
		local box =''
		if args.wikidata and string.match(cats,'missing linkback') then
			box = filepage_warningbox('missing_linkback', args.lang, args.wikidata)
		elseif args.wikidata and string.match(cats,'without home category') then
			box = filepage_warningbox('missing_homecat', args.lang, args.wikidata)
		end
		local doc = mw.getCurrentFrame():expandTemplate{ title ='documentation', args = { 'Template:Creator/documentation' } }
		results = results .. box .. doc -- add documentation to pages in creator namespace
	end
	return results
end

-- ===========================================================================
-- === This function is responsible for adding maintenance categories      ===
-- === which are not related to wikidata                                   ===
-- === INPUTS:                                                             ===
-- ===  * args  - merged data from the local arguments and Wikidata        ===
-- ===========================================================================
local function add_maintenance_categories(args)
	local cats = '' -- categories 
	
	-- ====================================================
	-- === automatic tagging of pages in all namespaces === 
	-- ====================================================
	if args.type=='' or args.type=='person' then
	    -- add an empty template which can be used as a tag in PetScan
		local dod  = args.deathyear or args.deathdate -- date of death
		local dob  = args.birthyear or args.birthdate -- date of birth
		local d    = os.date('!*t')                   -- current date table
		local year = tonumber(d.year)                 -- current year
		local pma  = nil                              -- years since death
		if dod then
			dod = tonumber(ISOdate._ISOyear(dod))
			if dod then
				pma = year-dod
			end
		end
		if dob and not pma then
			dob = tonumber(ISOdate._ISOyear(dob))
			if dob then
				pma = year-dob-100 -- Assumes max 100 lifespan
			end
		end

		-- Add empty tag templates to track different cases
		if pma and pma>100 then 
			mw.getCurrentFrame():expandTemplate{ title ='Works of authors who died more than 100 years ago' }
		elseif pma and pma>70 then 
			mw.getCurrentFrame():expandTemplate{ title ='Works of authors who died more than 70 years ago' }
		elseif (dod or dob or 0)>year-65 then 
			mw.getCurrentFrame():expandTemplate{ title ='Works of authors who died less than 65 years ago' }
		end
	end 
		
	-- ============================================================
	-- === automatic categorization of pages in File: namespace === 
	-- ============================================================
	if args.namespace==6 then
		if not args.image then
			mw.getCurrentFrame():expandTemplate{ title = 'Creator template without image' } -- add the template tag
		end
		return cats
	end
	
	-- ===============================================================
	-- === automatic categorization of pages in Creator: namespace === 
	-- ===============================================================
	if args.namespace~=100 then
		return cats
	end
	
	-- add [[Category:Creator templates]] category
	cats = cats .. string.format('\n[[Category:Creator templates|%s]]',args.sortkey or ' ')
	
	-- check for key information
	if not args.linkback and not args.wikidata then
		cats = cats .. '\n[[Category:Creator templates without linkback]]'
	end	
	if not args.name then
		cats = cats .. '\n[[Category:Creator templates without name]]'
	end	
	
	-- add homecat category
	if args.homecat then
		cats = cats .. string.format('\n[[Category:%s]]',args.homecat)
	end

	-- add type category
	if args.type then
		local lut = { 
			['commons user'] = '\n[[Category:User creator templates]]',
			['corporation']  = '\n[[Category:Corporate creator templates]]',
			['group']        = '\n[[Category:Group creator templates]]',
		}
		cats = cats ..  (lut[args.type] or '')
		if args.type=='commons user' then
			return cats -- for commons user do not add other maintenance categories
		end
	end
	
	-- ===============================================================
	-- === automatic categorization of pages in Creator: namespace === 
	-- === all pages except: 'commons user'                        ===
	-- ===============================================================
	
	-- check for image
	if not args.image then
		cats = cats .. '\n[[Category:Creator templates without images]]'
	end
	-- check for wikidata q-code
	if not args.wikidata then
		cats = cats .. '\n[[Category:Creator templates without Wikidata link]]'
	end
	-- check for homecat
	if not args.homecat then
		cats = cats .. '\n[[Category:Creator templates without home category]]'
	else
		local hc = mw.title.new('Category:'..args.homecat)
		if not hc.exists then
			cats = cats .. '\n[[Category:Creator templates with non-existing home categories]]'
		end 
		hc = mw.title.new('Creator:'..args.homecat)
		if hc:localUrl() ~= mw.title.getCurrentTitle():localUrl() then
			cats = cats .. '\n[[Category:Creator templates with non-matching home categories]]'
		end
	end

	return cats
end


-- ===========================================================================
-- === This function is responsible for adding maintenance categories      ===
-- === to pages in category namespace                                      ===
-- === INPUTS:                                                             ===
-- ===  * args - local inputs from the creator template page               ===
-- ===========================================================================
local function add_categories_to_category_namespace(args)
	local cats
	if args.namespace~=14 or (args.homecat and mw.title.new('Category:' .. args.homecat):localUrl() ~= mw.title.getCurrentTitle():localUrl()) then
		return '' -- if not a home category than exit
	end
	local sortkey = "|" .. (args.sortkey or '')
	if #sortkey==1 then sortkey='' end
	
	cats = string.format('\n[[Category:Creator template home categories%s]]', sortkey)
	-- check for wikidata q-code
	if not args.wikidata then
		cats = cats .. '\n[[Category:Creator template home categories without Wikidata link]]'
	end

	if args.command == 'autocategorize' then
		-- add basic categories to the creator page
		cats = string.format('%s\n[[Category:People by name%s]]', cats, sortkey)
		if args.deathyear then
			cats = string.format('%s\n[[Category:%i deaths%s]]', cats, args.deathyear, sortkey)
		end
		if args.birthyear then
			cats = string.format('%s\n[[Category:%i births%s]]', cats, args.birthyear, sortkey)
		end
	end

	return cats
end

-- ===========================================================================
-- === This function is responsible for adding maintenance categories      ===
-- === to pages in creator namespace which are related to wikidata         ===
-- === INPUTS:                                                             ===
-- ===  * args0 - local inputs from the creator template page              ===
-- ===  * args1 - merge of local and wikidata metadata                     ===
-- ===  * data  - data pulled from Wikidata                                ===
-- ===========================================================================
local function add_categories_to_creator_namespace(args0, args1, data)
	local cats = ''     -- categories 
	local qsTable = {}  -- table to store QuickStatements 
	local comp    = {}  -- outcome of argument vs. wikidata comparison
	-- two forms of QuickStatements command with and without quotes
	local qsCommand = {'%s|%s|%s', '%s|%s|"%s"'}

	-- compare Linkback to the actual page name. Many "Linkbacks" are created with 
	-- tool which produces &#38; and &#39;  instead of "&" and "'"
	if args0.linkback then
		local linkback = args0.linkback
		linkback = mw.ustring.gsub(linkback, '&#39;', "'")
		linkback = mw.ustring.gsub(linkback, '&#38;', "&")
		if linkback~=args0.pagename then
			cats = cats .. '\n[[Category:Creator templates with mismatching linkback]]'
		end
	end
	
	-- add [[Category:Creator templates with unknown parameter]] category, if some parameter not on the following list is used
	local fields = {'name', 'alternative_names', 'sortkey', 'birthdate', 'deathdate', 'birthloc', 'deathloc', 'workperiod', 'workloc', 'collapse', 
	        'image', 'homecat', 'nationality', 'gender', 'occupation', 'description', 'authority', 'type', 'wikisource', 'wikiquote', 'command',
			'namespace', 'linkback', 'wikidata', 'lang', 'pagename', 'reference', 'references', 'lifespan', 'birthyear', 'deathyear', 'option' }
	local set = {}
	for _, field in ipairs(fields) do set[field] = true end
	for field, _ in pairs( args0 ) do 
		if not set[field] then
			cats = string.format('%s\n[[Category:Creator templates with unknown parameter|%s]]', cats, field)
		end
	end
	
	-- add [[Category:Wikidata based creator templates]] and [[Category:Creator templates with Wikidata link: local linkback]]
	local val = {wikidata=1, option=0, linkback=0, lang=0, namespace=0, pagename=0, type=0, command=0 }
	local hash = 0;
	for field, _ in pairs( args0 ) do 
		hash = hash + (val[field] or 10)
	end
	if hash==1 then
		cats = string.format('%s\n[[Category:Creator templates based only on Wikidata|%s]]', cats, args1.sortkey or '')
	end
	
	--  if no q-code but we have "create" input argument then create new item
	if not args0.wikidata and args0.command == 'create item' then
		local description
		table.insert( qsTable, 'CREATE' )
		table.insert( qsTable, 'LAST|P31|Q5|S143|Q24731821' )  -- instance of human
		table.insert( qsTable, 'LAST|Len|"'.. args0.pagename .. '"' ) -- english label
		if args0.nationality and args0.occupation then
			local lang = args0.lang 
			args0.lang = 'en';
			description, _, _ = NationAndOccupation(args0)
			args0.lang = lang
			if args1.birthyear and args1.deathyear then
				description = string.format('%s (%s-%s)', description, args1.birthyear, args1.deathyear) 
			end
			table.insert( qsTable, 'LAST|Den|"'.. description .. '"' ) -- english description
		end
		args0.wikidata = 'LAST' 
	end
	
	-- skip the rest if no q-code
	if not args0.wikidata then
		return cats, args1
	end
	
	-- mark parameters as "local" if they are present in creator template
	local fields = {'name', 'birthdate', 'deathdate', 'birthyear', 'deathyear', 'birthloc', 'deathloc', 'image', 
		'homecat', 'nationality', 'gender', 'occupation', 'description', 'authority', 'wikisource', 'wikiquote', 'sortkey' }
	for _, field in ipairs( fields ) do
		if args0[field] then
			comp[field] = 'local'
		end
	end
	
	-- redundant if commons creator template and wikidata have those fields, without checking values
	local fields = {'wikiquote', 'wikisource' }
	for _, field in ipairs( fields ) do
		if args0[field] and data[field] then
			comp[field] = 'redundant'
		end
	end
	
	-- ==================================================
	-- === time fields  ================================= 
	-- ==================================================
	local fields = {birthdate='P569', deathdate='P570' }
	local a1, a2, d1, d2, dy
	for field, prop in pairs( fields ) do
		a1 = args0[field]        -- original creator template value often in iso (YYYY or YYYY-MM-DD) format 
		a2 = args1[field]        -- translated creator template value  
		d1 = data[field .. '_']  -- wikidata value in iso (YYYY or YYYY-MM-DD) format 
		d2 = data[field]         -- translated wikidata value
		dy = tostring(data[string.gsub(field, 'date', 'year')])  -- wikidata year value
		if a1 and not (string.match(a1,"^%d%d%d%d$") or string.match(a1,"^%d%d%d%d%-%d%d$") or string.match(a1,"^%d%d%d%d%-%d%d%-%d%d$")) then -- if YYYY or YYYY-MM-DD format 
			a1 = nil -- delete if not in iso format
		end
		
		if a1 then -- local date in iso form
			if (a1==d1) or (a2 and a2==d2) or (#a1==4 and a1==dy) then 
				comp[field] = 'redundant' -- matching iso value, translated value and commons-year matching wikidata date
			elseif d1 and a1~=d1 then 
				comp[field] = 'mismatching'
			elseif not d2 then -- missing on Wikidata 
				comp[field] = 'item missing'
			end
			-- create QS string so the Commons value can be uploded to Wikidata
			if (comp[field]=='item missing') or (#a1>4 and d1 and #d1==4 and string.sub(a1,1,4)==d1) then
				local val = isodate2timestamp(a1)
				if val then
					table.insert( qsTable, string.format(qsCommand[1], args0.wikidata, prop, val) )
				end
			end
		end
	end
	
	-- ==================================================
	-- === birthloc / deathloc place fields  ============ 
	-- ==================================================
	local fields = {birthloc='P19', deathloc='P20' }
	for field, prop in pairs( fields ) do
		local a1, a2, d1, d2, dy, _
		a2 = args0[field] -- creator template value  
		d1 = data[field]  -- wikidata q-code
		if a2 then
			a1, _ = City.qCode(a2) -- q-code for original creator template value
		end
		if d1 then
			d2 = getLabel(d1, 'en', '-') -- get english label
			dy = getLabel(d1, args0.lang) .. core.editAtWikidata(args0.wikidata, prop, args0.lang)
		end
		
		if (a1 and a1==d1) or (a2 and a2==d2)  then 
			comp[field] = 'redundant' -- matching q-code and name
		elseif (a1 and d1 and a1~=d1) or (a2 and d2 and a2~=d2) then 
			comp[field] = 'mismatching'
		elseif a1 and not d2 then -- missing on Wikidata 
			comp[field] = 'item missing'
			table.insert( qsTable, string.format(qsCommand[1], args0.wikidata, prop, a1) )
		elseif a2 and not d2 then
			comp[field] = 'item missing'
		end
		data [field..'_'] = d1 
		args0[field..'_'] = a1 
		data [field]      = dy
	end
	
	-- ==================================================
	-- === workloc field  =============================== 
	-- ==================================================
	if (args0.workloc and args0.workloc==data.workloc_en)  then 
		comp.workloc = 'redundant' -- matching q-code and name
	elseif args0.workloc and not data.workloc then -- missing on Wikidata 
		comp.workloc = 'item missing'
	end
	
	-- ==================================================
	-- === nationality and occupation  ==================
	-- ==================================================
	local fields = { nationality='nationality_', occupation='occupationEN' }
	data.nationality_ = data.nationality
	for field, field_ in pairs( fields ) do
		local a1, a2, ad
		a1 = args1[field_] -- creator template value  
		d1 = data [field_] -- wikidata q-code
		ad = intersect(a1, d1)
		
		if (a1 and d1 and #a1==#ad and (#d1==#ad or field=='occupation'))  then 
			-- for nationality all values on Commons must be the same as on Wikidata
			-- for occupation all commons values have to be on Wikidata but wikidata can have more than that
			comp[field] = 'redundant' 
		elseif (a1 and d1 and #a1>#ad) then 
			comp[field] = 'mismatching' -- some commons values are not on Wikidata
		elseif a1 and not d1 then -- missing on Wikidata 
			comp[field] = 'item missing'
		end
	end
 
	-- ==================================================
	-- === gender  =============================== 
	-- ==================================================	
	if args0.gender then -- look up q-codes of gender
		local GenderLut = { male='Q6581097', female='Q6581072'}
		a1 = GenderLut[mw.ustring.lower(args0.gender)] -- look up q-code for each gender
		d1 = GenderLut[data.gender]       -- wikidata q-code
		if a1 and d1 and a1~=d1 then 
			comp.gender = 'mismatching'
		elseif a1 and d1 and a1==d1 then 
			comp.gender = 'redundant'
		elseif a1 and not d1 then
			comp.gender = 'item missing'
			table.insert( qsTable, string.format(qsCommand[1], args0.wikidata, 'P21', a1) )		
		end	
	end
	
	-- ==================================================
	-- === odds and ends  =============================== 
	-- ==================================================	
	if args0.image then 
		args0.image_ = mw.uri.decode( args0.image, "WIKI" )
	end
	args0.linkback_ = args0.pagename;
	args0.homecat_  = args0.homecat;	
	
	local fields = {image='P18', linkback='P1472', homecat='P373'}
	for field, prop in pairs( fields ) do
		a1 = args0[field..'_'] -- creator template value  
		d1 = data[field]       -- wikidata q-code
		if a1 and d1 and a1~=d1 then 
			comp[field] = 'mismatching'
		elseif a1 and d1 and a1==d1 then 
			comp[field] = 'redundant'	
		elseif a1 and not d1 then
			comp[field] = 'item missing'
			table.insert( qsTable, string.format(qsCommand[2], args0.wikidata, prop, a1) )
		end	
	end
	if comp.linkback == 'redundant' and (hash~=1 or not args0.linkback) then
		comp.linkback = nil
	end
	if args0.sortkey and data.sortkey and args0.sortkey==data.sortkey then
		comp.sortkey = 'redundant'
	end
	if args0.description and args1.description_==args0.description then -- description is "French painter" while nationality is FR and occupation is "painter"
		comp.description = 'redundant'
	end
	
	-- ==================================================
	-- === alter look of some fields === 
	-- ==================================================
	local fields = {'birthloc', 'deathloc', 'birthdate', 'deathdate' }
	for _, field in ipairs( fields ) do
		if ( comp[field] == 'mismatching' ) or ( comp[field] == 'local' and data[field] ) then
			args1[field] = string.format('<div style=\"background-color:PeachPuff\">%s</div> <br/>%s', args1[field], data[field])
		elseif ( comp[field] == 'redundant' ) then
			args1[field] = string.format('<div style=\"background-color:Thistle\">%s</div>', args1[field])
		elseif ( comp[field] == 'item missing' and args1[field])  then
			args1[field] = string.format('<div style=\"background-color:PeachPuff\">%s</div>', args1[field])
		end
	end

	-- ==================================================
	-- === Create categories and QuickStatement codes === 
	-- ==================================================
	
	-- create categories based on comp structure
	for field, outcome in pairs( comp ) do
		cats = string.format('%s\n[[Category:Creator templates with Wikidata link: %s %s]]', cats, outcome, field)
	end
	
	-- convert QS table to a string
	local QS   = ''     -- quick_statements final string
	if #qsTable>0 then
		local today = '+' .. os.date('!%F') .. 'T00:00:00Z/11' -- today's date in QS format
	    local url   = mw.title.getCurrentTitle():canonicalUrl()
	    local source = '|S143|Q24731821|S813|' .. today .. '|S4656|"' .. url .. '"'
		local qsWrapper = '&nbsp;[[File:Commons_to_Wikidata_QuickStatements.svg|15px|link=%s]]'
		QS = table.concat( qsTable, source..'||') .. source    -- combine multiple statements into a single command separated by ||
		QS = mw.ustring.gsub(QS, ' ', "%%20")
		QS = mw.ustring.gsub (mw.uri.encode(QS),'%%2520','%%20')
		QS = 'https://quickstatements.toolforge.org/#/v1=' .. QS    -- create full URL link
		QS = string.format(qsWrapper, QS)
		cats = cats .. '\n[[Category:Creator templates with Wikidata link: quick statements]]'
	end
	args1.QS = QS
	
	-- tag invalid dates that use YYYY-MM-DD format
	if (args0.deathdate and (not valid_date(args0.deathdate))) or 
	   (args0.birthdate and (not valid_date(args0.birthdate))) then
		cats = cats .. '\n[[Category:Pages with incorrect date]]'
	end
	
	return cats, args1
end


-- ===========================================================================
-- === Harvest wikidata properties matching creator template fields ==========
-- ===========================================================================

local function getPropertyQual(entity, prop, qualifiers, lang, offset)
	local Res = {}
	if entity.claims and entity.claims[prop] then
		for k, statement in ipairs( entity:getBestStatements( prop )) do
			if (statement.mainsnak.snaktype == "value") then 
				local res = {} -- table with fields: key, value, P... (qualifiers)
				local jdn = k -- "Julian day number" will be used as a key for sorting events; initialize
				local val = statement.mainsnak.datavalue.value.id
				val = getLabel(val, lang)
				res.value = val
				for iQual, qual in ipairs( qualifiers ) do
					if statement.qualifiers and statement.qualifiers[qual] then
						local snak = statement.qualifiers[qual][1]
						if (snak.snaktype == "value" and snak.datatype == 'time') then
							val = qualifierDate(snak, lang)
							if iQual==1 then -- first qualifier in the qualifiers list will be used as a sorting value
								jdn = val.jdn
							end
							val = val.str
						end
						res[qual] = val
					end
				end
				res.key = jdn
				table.insert(Res, res)
			end
		end
	end
	local tableComp = function (rec1, rec2) return rec1.key<rec2.key end
	table.sort(Res, tableComp)
	return Res
end

-- ===========================================================================
local function get_work_location(entity, lang)
	-- work_location (P937) /  'P580', 'P582' (time properties) 
	local prop = getPropertyQual(entity, 'P937', {'P580', 'P582', 'P585'}, lang)
	local X={}
	for _, p in ipairs(prop) do
		local str = p.value
		if p.P580 or p.P582 then
			str = string.format("%s (%s&ndash;%s)", p.value, p.P580 or '', p.P582 or '')
		elseif p.P585 then
			str = string.format("%s (%s)", p.value, p.P585)
		else
			str = p.value
		end
		table.insert(X, str)
	end
	if #X>0 then
		return table.concat(X,"; ")
	end
	return nil
end

-- ===========================================================================
local function editAtWikidata(data, qCode, property,  lang)
	for prop, field in pairs( property ) do
		 if data[field] then
			data[field] = data[field].. core.editAtWikidata(qCode, prop,  lang)
		 end
	end
	return data
end

-- ===========================================================================
local function harvest_wikidata(qCode, lang, namespace, pagename)
	local str, d
	local data = {} -- structure similar to "args" but filled with wikidata data
	local cats = ''
	local entity = nil
	if mw.wikibase and qCode then
		entity = mw.wikibase.getEntity(qCode)
		if not entity then
			 cats = '[[Category:Creator templates with bad Wikidata link|invalid]]' 
 		elseif entity.id~=qCode then
			 cats = '[[Category:Creator templates with redirected Wikidata link]]' 
			 qCode = entity.id
		end
	end
	if not entity then
		return data, cats
	end

	-- ===========================================================================
	-- === Step 1: time properties
	-- ===========================================================================	
	-- harvest time properties: translated date and year number
	local d1 = getDate(entity, 'P569' , lang)
	local d2 = getDate(entity, 'P570' , lang)
	local d3 = getDate(entity, 'P1636', lang)	
	local d4 = getDate(entity, 'P4602', lang)	
	data.birthdate, data.birthdate_, data.birthyear = d1.str, d1.iso, d1.year
	data.deathdate, data.deathdate_, data.deathyear = d2.str, d2.iso, d2.year
	data.baptism,   data.baptismyear                = d3.str,         d3.year
	data.burial,    data.burialyear                 = d4.str,         d4.year
	data = editAtWikidata(data, qCode, { P569='birthdate', P570='deathdate'},  lang)
	
	-- baptism date as birth date
	if not data.birthdate and data.baptism then 
		data.birthdate = mw.getCurrentFrame():expandTemplate{ title='Lifetime date', args={'baptism', data.baptism, lang=lang} } .. core.editAtWikidata(qCode, 'P1636', lang)
		data.birthyear = data.baptismyear
	end
	-- burial date as death date
	if not data.birthdate and data.baptism then 
		data.deathdate = mw.getCurrentFrame():expandTemplate{ title='Lifetime date', args={'buried', data.burial, lang=lang} } .. core.editAtWikidata(qCode, 'P4602', lang)
		data.deathyear = data.burialyear
	end
	data.birthyear = tostring(data.birthyear or '')
	data.deathyear = tostring(data.deathyear or '')

	-- workperiod
	local property = { P2031='workperiod1', P2032='workperiod2', P1317='workperiod'}
	for prop, field in pairs( property ) do
		 d1 = getDate(entity, prop, lang)
		 if d1.str then
			data[field..'_'] = d1.str
			data[field] = d1.str .. core.editAtWikidata(qCode, prop,  lang)
		 end
	end
	if not data.workperiod and (data.workperiod1 or data.workperiod2) then 
		data.workperiod = (data.workperiod1  or '') .. '&ndash;' .. (data.workperiod2  or '')
		data.workperiod_= (data.workperiod1_ or '') .. '&ndash;' .. (data.workperiod2_ or '')
	end
	data.workloc    = get_work_location(entity, lang) 
	if data.workloc then
		data.workloc = data.workloc .. core.editAtWikidata(qCode, 'P937',  lang)
	end
	data.workloc_en = get_work_location(entity, 'en')
	
	-- lifespan displayed after name
	if data.birthyear~='' or data.deathyear~='' then 
		data.lifespan = string.format('(%s&ndash;%s)', data.birthyear, data.deathyear)
	elseif data.workperiod_ then -- create from work period (without "edit at wikidata")
		data.lifespan = string.format('([[d:Q36424|fl.]] %s)', data.workperiod_)  
	end
		
	-- ===========================================================================
	-- === Step 2: simple string and Q-code properties
	-- ===========================================================================	
	-- harvest string and Q-code properties
	local property = {P18='image', P19='birthloc', P20='deathloc',
		P109='signature', P373='homecat', P734='lastname', P735='firstname',
		P742='pseudonym', P1472='linkback', P1477='birthname',
		P1782='courtesyname', P1787='artname'}
	for prop, field in pairs( property ) do
		if entity.claims and entity.claims[prop] then -- if we have wikidata item and item has the property
			-- capture single "best" Wikidata value
			for _, statement in pairs( entity:getBestStatements( prop )) do
				if (statement.mainsnak.snaktype == 'value') then
					local v = statement.mainsnak.datavalue.value
					if v.id then v = v.id end
					data[field] = v
				end
			end
		end
	end
	
	-- if homecat missing than look for it among sitelinks
	if not data.homecat then
		local sitelink
		if entity.sitelinks and entity.sitelinks.commonswiki then
			sitelink = entity.sitelinks.commonswiki.title
		end
		if not sitelink then -- check for "topic's main category" sitelinks
			local cat_id = (core.parseStatements(entity:getBestStatements( "P910" ), nil) or {nil})[1]
			if cat_id then
				sitelink = Wikidata_label._sitelinks(cat_id, "commons")['']
			end
		end		
		if sitelink and mw.ustring.sub(sitelink,1,9)=='Category:' then
			data.homecat = mw.ustring.sub(sitelink,10)
		end
	end
	
	-- get "sortkey" field
	if not data.sortkey then
		local lastname, firstname, name_part
		if data.lastname then
			lastname = getLabel(data.lastname, lang, '-')
		elseif namespace == 100 then
			name_part = mw.text.split(pagename, '%(')
			name_part = mw.text.trim (name_part[1])
			name_part = mw.text.split(name_part, ' ')
			lastname  = name_part[#name_part]
		else
			lastname  = "ZZZ"
		end
		data.lastname = lastname
		if data.firstname then
			firstname = getLabel(data.firstname, lang, '-')
		else
			firstname = data.linkback or ''
		end
		data.sortkey =  lastname .. ', ' .. firstname 
	end
	
	-- convert gender
	data.gender_ = data.gender
	if data.gender=='Q6581097' or data.gender=='Q2449503' then
		data.gender = 'male'
	end
	if data.gender=='Q6581072' or data.gender=='Q1052281' then
		data.gender = 'female'
	end
	data.image = data.image or data.signature
	
	-- =================================================================================
	-- === Step 5: name, wikisource, wikiquote, alternative_names and authority control
	-- =================================================================================	
	-- get name field
	data.name = getLabel(entity, lang, 'wikipedia') -- create name based on wikidata label

	-- prepare fallback list of languages
	local langList = mw.language.getFallbacksFor(lang)
	table.insert(langList, 1, lang)

	-- get alternative names
	local altNameTable = {}
	local nameTypes = {birthname='birth', pseudonym='pseudonym',
		courtesyname='courtesy name', artname='artist'}
	for t,operation in pairs(nameTypes) do
		if data[t] then
			if data[t].text then
				data[t] = data[t].text
			end
			local nameStr = alterName(operation, data[t], lang)
			table.insert(altNameTable, nameStr)
		end
	end
	for _, lng in ipairs(langList) do 
		local aliasTable = Wikidata_label._aliases(entity, lng)
		if #aliasTable>0 and #aliasTable<8 then -- skip aliases if more than 8 of them
			for _,alias in pairs(aliasTable) do
				local isSpecialName = false
				for t,_ in pairs(nameTypes) do
					if alias == data[t] then
						isSpecialName = true
					end
				end
				if not isSpecialName then
					table.insert(altNameTable, alias)
				end
			end
			data.alternative_names = table.concat(altNameTable, '; ')
			break 
		end
	end
	
	-- get wikisource and wikiquote link
	local projects = {s='wikisource', q='wikiquote'}
	for code, project in pairs(projects) do
		local sitelinks = Wikidata_label._sitelinks(entity, project)
		if sitelinks then
			local lng, _ = next(sitelinks)    -- get language of the first sitelink
			table.insert(langList, lng) -- and add it to the list	so there is at least one lang with sitelink on the list
			for _, language in ipairs(langList) do 
				local sitelink = sitelinks[language]
				if sitelink then 
					data[project] = string.format('%s:%s:%s', code, language, sitelink)
					break 
				end
			end
		end
	end			
	
	-- get authority control template		
	local AC_cats
	local nIdent = nil -- number of authority control identifiers to display (nil means unlimited)
	if namespace == 6 then 
		nIdent = 5     -- limit number of identifiers in file namespace for clarity
	end
	data.authority, AC_cats = authorityControl(entity, {wikidata = qCode}, lang, nIdent) 
	if not (namespace == 2 or namespace == 6 or namespace == 828 or math.fmod(namespace,2)==1) then
		cats = cats .. AC_cats -- lets not add authorityControl categories to user pages, files, modules or talk pages and concentrate on templates and categories instead
	end
	
	return data, cats
end

-- ==================================================
-- === External functions ===========================
-- ==================================================
local p = {}

-- ===========================================================================
-- === Version of the function to be called from other LUA codes
-- ===========================================================================
function p._creator(args)
	local lang = args.lang  -- user's language
	local cats = ''         -- categories 
	local str, data
	
	-- look up title info
	local title    = mw.title.getCurrentTitle()
	args.namespace = title.namespace   -- get page namespace
	args.pagename  = title.text        -- get {{PAGENAME}}
	
	-- ===========================================================================
	-- === Step 1: clean up of template arguments "args"
	-- ===========================================================================
	args.type = string.lower(args.type or 'person')   -- if 'type' field is not specified than set to "person"
	if args.linkback then
		args.linkback = string.sub(args.linkback,9)
	end

	-- clean up "gender" field
	if string.sub(args.gender or '',1,1)=='m' then args.gender=  'male' end
	if string.sub(args.gender or '',1,1)=='f' then args.gender='female' end
	
	--make a copy of args structure to capture raw inputs
	local args0 = {} -- original args
	for name, value in pairs( args ) do 
		args0[name] = value
	end	
	
	--get birthyear and deathyear from full dates
	if args.birthdate then
		args.birthyear = empty2nil(ISOdate._ISOyear(args.birthdate))
		args.birthdate = ISOdate._ISOdate(args.birthdate, lang)
	end
	if args.deathdate then
		args.deathyear = empty2nil(ISOdate._ISOyear(args.deathdate))
		args.deathdate = ISOdate._ISOdate(args.deathdate, lang)
	end
	
	-- ===========================================================================
	-- === Step 2: one by one merge wikidata and creator data
	-- ===========================================================================
	data, cats = harvest_wikidata(args.wikidata, lang, args.namespace, args.pagename)
	local description, args1, data1 = NationAndOccupation(args)
	local fields = {'nationality', 'occupation', 'gender', 'occupationEN'}
	for _, field in ipairs( fields ) do 
		args[field] = args1[field] 
		data[field] = data1[field]
	end
	args.nationality_ = args.nationality
	
	-- mass merge (prioritize local values)
	fields = {'name', 'alternative_names', 'sortkey', 'birthdate', 'deathdate', 'birthloc', 'deathloc', 'workperiod',
	        'image', 'homecat', 'nationality', 'gender', 'occupation', 'authority',  'wikisource', 'wikiquote', 'workloc', 
					'linkback', 'lifespan', 'birthyear', 'deathyear', 'collapse' }
	for _, field in ipairs( fields ) do 
		args[field] = args[field] or data[field]
	end
	
	-- process "name" field
	if args.option and args.option~='' then -- modify name based on "option" parameter
		local base_name = args.name
		-- call [[module:Name]] with the task
		args.name = alterName(args.option, args.name, lang)
		if args.name  == "name not supported" then 
			args.name = base_name
			cats = cats .. '\n[[Category:Bad use of creator template - option]]'
		end
	end
		
	-- process places fields
	-- locations can be words or q -codes. Add links 
	args.birthloc = City._city(args.birthloc, lang) 
	args.deathloc = City._city(args.deathloc, lang)
	if args.workloc and not string.find(args.workloc, ' ') then
		args.workloc = City._city(args.workloc, lang) -- single word workloc will get a link
	end
	
	-- lifespan displayed after name
	if args.lifespan then
		args.lifespan = string.gsub(args.lifespan, '-', '&ndash;') -- use special dash
	end

	-- process "Authority Control" field
	args.authority_tag = getLabel("Q36524", args.lang, "wikipedia", "ucfirst")
	
	-- process "description" field
	-- Add phrase like "French painter" to the description field
	description = mw.text.trim(description)
	if description and #description>0 then
		if args.description then
			args.description_=  description
			args.description =  description .. '<br/>' .. args.description
		else
			args.description =  description
		end
	end
		
	-- use Normalization Form D to convert string with accented characters to more sort friendly format
	-- See http://unicode.org/reports/tr15/ for examples
	args.sortkey  = mw.ustring.toNFD(args.sortkey or '')
	
	-- references are only shown in ''Creator'' namespace
	if args.namespace~=100 then 
		args.references = nil
	end
	
	-- convert all empty strings to nils
	for _, field in ipairs( fields ) do 
		if args[field] == '' then 
			args[field] = nil; 
		end
	end
	
	-- ===========================================================================
	-- === Step 3: create maintenance categories and render html of the table
	-- ===========================================================================
	if args.namespace==14 and (args.type=='' or args.type=='person') then
		cats = cats .. add_categories_to_category_namespace(args)
	end
	cats = cats .. add_maintenance_categories(args)
	
	-- If creator namespace and "person" template than add maintenance categories
	args.QS = nil;
	if args.namespace==100 and (args.type=='' or args.type=='person') then
		str, args = add_categories_to_creator_namespace(args0, args, data)
		cats = cats .. str
	end
	local results = Build_html(args, cats)
	return results, cats
end

-- ===========================================================================
-- === Version of the function to be called from template namespace
-- ===========================================================================
function p.creator(frame)
	-- switch to lowercase parameters to make them case independent
	local args = core.getArgs(frame)

	-- alias field names
	args.references = args.references or args.reference      -- two alternative names for references
	
	-- parse args.option field, which is passed through individual Creator template (page in Creator namespace)
	local options = mw.text.split(args.option or '', '/') -- individual keywords can be separated by "/"
	args.option = nil
	for _, option in pairs( options ) do
		if option == 'autocategorize' then
			args.command = option -- some "options" are to modify the name and some are commands to do things
		elseif option == 'collapse' then
			args.collapse = 1 -- some "options" are to modify the name and some are commands to do things
		elseif #option>3 then
			args.option = option
		end
	end
	if args.wikidata == "create" then
		args.command  = "create item"
		args.wikidata = nil
	end
	
	-- Create invisible language independent marking in format similar to QuickStatements code based on Wikidata and Option
	local QS = ''
	if args.wikidata and string.match(args.wikidata or '', "^Q%d+$") then -- invisible language independent marking
		if not args.option then -- no "option" modifier
			QS = TagQS.createTag('creator', 'P170', args.wikidata)
		else
			args.option = mw.ustring.gsub(mw.ustring.lower(args.option), '_', ' ') -- normalize input string
		    -- Not handled options: "studio of", "or follower", "or workshop", "and workshop",  "formerly attributed to"
			local qual1 = {['workshop of']='P1774', ['follower of']='P1775', ['circle of']='P1776', 
		          ['manner of']='P1777', ['school of']='P1780', ['after']='P1877'} -- ['possibly']='P1779', 
			local qual2 = {['probably']='Q56644435', ['presumably']='Q18122778', ['possibly']='Q30230067', ['attributed to']='Q230768'}
			if qual1[args.option] then -- use anonymous = Q4233718 with qualifier
				QS = TagQS.createTag('creator', 'P170', string.format('Q4233718,%s,%s', qual1[args.option], args.wikidata))
			elseif qual2[args.option] then -- use "nature of statement" (P5102) qualifier
				QS = TagQS.createTag('creator', 'P170', string.format('%s,P5102,%s', args.wikidata, qual2[args.option]))
			end
		end
	end
	
	-- call the inner "core" function
	local results, cats = p._creator(args)	
	return results .. QS .. cats
end

return p