-- ------------------------------------------------------------------------------ --
--                                TradeSkillMaster                                --
--                http://www.curse.com/addons/wow/tradeskill-master               --
--                                                                                --
--             A TradeSkillMaster Addon (http://tradeskillmaster.com)             --
--    All Rights Reserved* - Detailed license information included with addon.    --
-- ------------------------------------------------------------------------------ --

TSMAPI_FOUR.Database = {}
local private = { recycledRows = {}, recycledQueries = {}, recycledQueryClauses = {}, sortContext = nil }
local VALID_TYPES = {
	boolean = true,
	string = true,
	number = true,
}



-- ============================================================================
-- DatabaseRow Class Definitions
-- ============================================================================

local DatabaseRow = TSMAPI_FOUR.Class.DefineClass("DatabaseRow")

function DatabaseRow.__init(self)
	self._db = nil
	self._created = false
	self._noInsert = false
	self._pendingChanges = {}
end

function DatabaseRow._Acquire(self, db)
	self._db = db
	self._created = false
	self._noInsert = false
end

function DatabaseRow._Release(self)
	self._db = nil
	assert(not next(self._pendingChanges))
end

function DatabaseRow.SetField(self, field, value)
	local fieldType = self._db:_GetFieldType(field)
	if not fieldType then
		error(format("Field %s doesn't exist", tostring(field)), 3)
	elseif fieldType ~= type(value) then
		error(format("Field %s should be a %s, got %s", tostring(field), tostring(self._db:_GetFieldType(field)), type(value)), 3)
	end
	if value == self[field] then
		-- setting the field to its original value, so clear any pending change
		self._pendingChanges[field] = nil
	else
		self._pendingChanges[field] = value
	end
	return self
end

function DatabaseRow.IncrementField(self, field)
	local fieldType = self._db:_GetFieldType(field)
	assert(self._db:_GetFieldType(field) == "number", format("Cannot increment a field of type %s", fieldType))
	self._pendingChanges[field] = self[field] + 1
	return self
end

function DatabaseRow.DecrementField(self, field)
	local fieldType = self._db:_GetFieldType(field)
	assert(self._db:_GetFieldType(field) == "number", format("Cannot increment a field of type %s", fieldType))
	self._pendingChanges[field] = self[field] - 1
	return self
end

function DatabaseRow.GetField(self, field)
	local fieldType = self._db:_GetFieldType(field)
	if not fieldType then
		error(format("Field %s doesn't exist", tostring(field)))
	end
	return self[field]
end

function DatabaseRow.GetFields(self, ...)
	local numFields = select("#", ...)
	local field1, field2, field3, field4, field5, field6, field7, field8 = ...
	if numFields == 0 then
		return
	elseif numFields == 1 then
		return self:GetField(field1)
	elseif numFields == 2 then
		return self:GetField(field1), self:GetField(field2)
	elseif numFields == 3 then
		return self:GetField(field1), self:GetField(field2), self:GetField(field3)
	elseif numFields == 4 then
		return self:GetField(field1), self:GetField(field2), self:GetField(field3), self:GetField(field4)
	elseif numFields == 5 then
		return self:GetField(field1), self:GetField(field2), self:GetField(field3), self:GetField(field4), self:GetField(field5)
	elseif numFields == 6 then
		return self:GetField(field1), self:GetField(field2), self:GetField(field3), self:GetField(field4), self:GetField(field5), self:GetField(field6)
	elseif numFields == 7 then
		return self:GetField(field1), self:GetField(field2), self:GetField(field3), self:GetField(field4), self:GetField(field5), self:GetField(field6), self:GetField(field7)
	elseif numFields == 8 then
		return self:GetField(field1), self:GetField(field2), self:GetField(field3), self:GetField(field4), self:GetField(field5), self:GetField(field6), self:GetField(field7), self:GetField(field8)
	else
		error("GetFields() only supports up to 8 fields")
	end
end

function DatabaseRow.Save(self)
	assert(not self._noInsert)
	local oldValues = nil
	-- apply all the pending changes
	for field, value in pairs(self._pendingChanges) do
		if self._created then
			oldValues = oldValues or TSMAPI_FOUR.Util.AcquireTempTable()
			oldValues[field] = self[field]
		end
		self[field] = value
	end
	wipe(self._pendingChanges)

	-- go through and handle the extensions
	for _, callback in ipairs(self._db._extensionCallbacks) do
		callback(self)
	end

	-- apply all the pending changes from the extensions
	for field, value in pairs(self._pendingChanges) do
		if self._created then
			oldValues = oldValues or TSMAPI_FOUR.Util.AcquireTempTable()
			oldValues[field] = self[field]
		end
		self[field] = value
	end
	wipe(self._pendingChanges)

	if not self._created then
		self._created = true
		self._db:_InsertRow(self)
	elseif oldValues then
		self._db:_UpdateRow(self, oldValues)
		TSMAPI_FOUR.Util.ReleaseTempTable(oldValues)
	end

	return self
end

function DatabaseRow.SaveNoInsert(self)
	assert(not self._db:_GetRowIndex(self))
	-- apply all the pending changes
	for field, value in pairs(self._pendingChanges) do
		self[field] = value
	end
	wipe(self._pendingChanges)
	self._noInsert = true
	return self
end

function DatabaseRow.Discard(self)
	assert(self._noInsert)
	self:_Release()
	tinsert(private.recycledRows, self)
end

function DatabaseRow.CalculateHash(self, fields)
	local hash = nil
	for _, field in ipairs(fields) do
		hash = TSMAPI_FOUR.Util.CalculateHash(self[field], hash)
	end
	return hash
end



-- ============================================================================
-- DatabaseQueryClause Class Definitions
-- ============================================================================

local DatabaseQueryClause = TSMAPI_FOUR.Class.DefineClass("DatabaseQueryClause")

function DatabaseQueryClause.__init(self)
	self._query = nil
	self._operation = nil
	self._parent = nil
	-- equal
	self._field = nil
	self._value = nil
	-- or / and
	self._subClauses = nil
end

function DatabaseQueryClause._Acquire(self, query, parent)
	self._query = query
	self._parent = parent
end

function DatabaseQueryClause._Release(self)
	self._query = nil
	self._operation = nil
	self._parent = nil
	self._field = nil
	self._value = nil
	if self._subClauses then
		for _, clause in ipairs(self._subClauses) do
			clause:_Release()
		end
		TSMAPI_FOUR.Util.ReleaseTempTable(self._subClauses)
		self._subClauses = nil
	end
	tinsert(private.recycledQueryClauses, self)
end

function DatabaseQueryClause._GetParent(self)
	return self._parent
end

function DatabaseQueryClause._IsTrue(self, row)
	if self._operation == "EQUAL" then
		return row:GetField(self._field) == self._value
	elseif self._operation == "NOT_EQUAL" then
		return row:GetField(self._field) ~= self._value
	elseif self._operation == "LESS" then
		return row:GetField(self._field) < self._value
	elseif self._operation == "LESS_OR_EQUAL" then
		return row:GetField(self._field) <= self._value
	elseif self._operation == "GREATER" then
		return row:GetField(self._field) > self._value
	elseif self._operation == "GREATER_OR_EQUAL" then
		return row:GetField(self._field) >= self._value
	elseif self._operation == "MATCHES" then
		return strmatch(strlower(row:GetField(self._field)), self._value) and true or false
	elseif self._operation == "CUSTOM" then
		return self._field(row, self._value) and true or false
	elseif self._operation == "HASH_EQUAL" then
		return row:CalculateHash(self._field) == self._value
	elseif self._operation == "OR" then
		for _, subClause in ipairs(self._subClauses) do
			if subClause:_IsTrue(row) then
				return true
			end
		end
		return false
	elseif self._operation == "AND" then
		for _, subClause in ipairs(self._subClauses) do
			if not subClause:_IsTrue(row) then
				return false
			end
		end
		return true
	else
		error("Invalid operation: " .. tostring(self._operation))
	end
end

function DatabaseQueryClause._GetIndexValue(self, indexField)
	if self._operation == "EQUAL" then
		if self._field ~= indexField then
			return nil
		end
		return self._value
	elseif self._operation == "AND" then
		if #self._subClauses ~= 1 then
			return nil
		end
		return self._subClauses[1]:_GetIndexValue(indexField)
	end
end

function DatabaseQueryClause._InsertSubClause(self, subClause)
	assert(self._operation == "OR" or self._operation == "AND")
	tinsert(self._subClauses, subClause)
	return self
end

function DatabaseQueryClause._SetComparisonOperation(self, operation, field, value)
	assert(not self._operation)
	self._operation = operation
	self._field = field
	self._value = value
	return self
end

function DatabaseQueryClause._SetSubClauseOperation(self, operation)
	assert(not self._operation)
	self._operation = operation
	self._subClauses = TSMAPI_FOUR.Util.AcquireTempTable()
	return self
end

function DatabaseQueryClause.Equal(self, field, value)
	return self:_SetComparisonOperation("EQUAL", field, value)
end

function DatabaseQueryClause.NotEqual(self, field, value)
	return self:_SetComparisonOperation("NOT_EQUAL", field, value)
end

function DatabaseQueryClause.LessThan(self, field, value)
	return self:_SetComparisonOperation("LESS", field, value)
end

function DatabaseQueryClause.LessThanOrEqual(self, field, value)
	return self:_SetComparisonOperation("LESS_OR_EQUAL", field, value)
end

function DatabaseQueryClause.GreaterThan(self, field, value)
	return self:_SetComparisonOperation("GREATER", field, value)
end

function DatabaseQueryClause.GreaterThanOrEqual(self, field, value)
	return self:_SetComparisonOperation("GREATER_OR_EQUAL", field, value)
end

function DatabaseQueryClause.Matches(self, field, value)
	return self:_SetComparisonOperation("MATCHES", field, value)
end

function DatabaseQueryClause.Custom(self, func, arg)
	return self:_SetComparisonOperation("CUSTOM", func, arg)
end

function DatabaseQueryClause.HashEqual(self, fields, value)
	return self:_SetComparisonOperation("HASH_EQUAL", fields, value)
end

function DatabaseQueryClause.Or(self)
	return self:_SetSubClauseOperation("OR")
end

function DatabaseQueryClause.And(self)
	return self:_SetSubClauseOperation("AND")
end



-- ============================================================================
-- DatabaseQuery Class Definitions
-- ============================================================================

local DatabaseQuery = TSMAPI_FOUR.Class.DefineClass("DatabaseQuery")

function DatabaseQuery.__init(self)
	self._db = nil
	self._rootClause = nil
	self._currentClause = nil
	self._orderBy = {}
	self._orderByAscending = {}
	self._distinct = nil
	self._updateCallback = nil
	self._select = {}
	self._iterReadOnlyState = nil
end

function DatabaseQuery._Acquire(self, db)
	self._db = db
	-- implicit root AND clause
	self._rootClause = private.NewDatabaseQueryClause(self)
		:And()
	self._currentClause = self._rootClause
end

function DatabaseQuery._Release(self)
	assert(self._iterReadOnlyState == nil)
	-- remove from the database
	self._db:_RemoveQuery(self)
	self._db = nil
	self._rootClause:_Release()
	self._rootClause = nil
	self._currentClause = nil
	wipe(self._orderBy)
	wipe(self._orderByAscending)
	self._distinct = nil
	self._updateCallback = nil
	wipe(self._select)
end

function DatabaseQuery._DataUpdated(self)
	if self._updateCallback then
		self:_updateCallback()
	end
end

function DatabaseQuery._NewClause(self)
	local newClause = private.NewDatabaseQueryClause(self, self._currentClause)
	self._currentClause:_InsertSubClause(newClause)
	return newClause
end

function DatabaseQuery.Release(self)
	self:_Release()
	tinsert(private.recycledQueries, self)
end

function DatabaseQuery.HasField(self, field)
	return self._db:_GetFieldType(field) and true or false
end

function DatabaseQuery.Equal(self, field, value)
	assert(self._db:_GetFieldType(field) == type(value))
	self:_NewClause()
		:Equal(field, value)
	return self
end

function DatabaseQuery.NotEqual(self, field, value)
	assert(self._db:_GetFieldType(field) == type(value))
	self:_NewClause()
		:NotEqual(field, value)
	return self
end

function DatabaseQuery.LessThan(self, field, value)
	assert(self._db:_GetFieldType(field) == type(value))
	self:_NewClause()
		:LessThan(field, value)
	return self
end

function DatabaseQuery.LessThanOrEqual(self, field, value)
	assert(self._db:_GetFieldType(field) == type(value))
	self:_NewClause()
		:LessThanOrEqual(field, value)
	return self
end

function DatabaseQuery.GreaterThan(self, field, value)
	assert(self._db:_GetFieldType(field) == type(value))
	self:_NewClause()
		:GreaterThan(field, value)
	return self
end

function DatabaseQuery.GreaterThanOrEqual(self, field, value)
	assert(self._db:_GetFieldType(field) == type(value))
	self:_NewClause()
		:GreaterThanOrEqual(field, value)
	return self
end

function DatabaseQuery.Matches(self, field, value)
	assert(self._db:_GetFieldType(field) == "string" and type(value) == "string")
	self:_NewClause()
		:Matches(field, strlower(value))
	return self
end

function DatabaseQuery.Custom(self, func, arg)
	assert(type(func) == "function")
	self:_NewClause()
		:Custom(func, arg)
	return self
end

function DatabaseQuery.HashEqual(self, fields, value)
	assert(type(fields) == "table")
	for _, field in ipairs(fields) do
		local fieldType = self._db:_GetFieldType(field)
		if not fieldType then
			error(format("Field %s doesn't exist", tostring(field)))
		elseif fieldType ~= "number" and fieldType ~= "string" then
			error(format("Cannot hash field of type %s", fieldType))
		end
	end
	self:_NewClause()
		:HashEqual(fields, value)
	return self
end

function DatabaseQuery.And(self)
	self._currentClause = self:_NewClause()
		:And()
	return self
end

function DatabaseQuery.Or(self)
	self._currentClause = self:_NewClause()
		:Or()
	return self
end

function DatabaseQuery.End(self)
	assert(self._currentClause ~= self._rootClause, "No current clause to end")
	self._currentClause = self._currentClause:_GetParent()
	assert(self._currentClause)
	return self
end

function DatabaseQuery.OrderBy(self, field, ascending)
	assert(ascending == true or ascending == false)
	local fieldType = self._db:_GetFieldType(field)
	if not fieldType then
		error(format("Field %s doesn't exist", tostring(field)))
	elseif fieldType ~= "number" and fieldType ~= "string" and fieldType ~= "boolean" then
		error(format("Cannot order by field of type %s", tostring(fieldType)))
	end
	tinsert(self._orderBy, field)
	tinsert(self._orderByAscending, ascending)
	return self
end

function DatabaseQuery.Distinct(self, field)
	assert(self._db:_GetFieldType(field), format("Field %s doesn't exist", tostring(field)))
	self._distinct = field
	return self
end

function DatabaseQuery.Select(self, ...)
	assert(#self._select == 0)
	local numFields = select("#", ...)
	assert(numFields > 0, "Must select at least 1 field")
	-- DatabaseRow.GetFields() only supports 8 fields, so we can only support 8 here as well
	assert(numFields <= 8, "Select() only supports up to 8 fields")
	for i = 1, numFields do
		local field = select(i, ...)
		assert(self._db:_GetFieldType(field), format("Field %s doesn't exist", tostring(field)))
		tinsert(self._select, field)
	end
	return self
end

function DatabaseQuery.SetUpdateCallback(self, func)
	self._updateCallback = func
	return self
end

function DatabaseQuery.Iterator(self, isReadOnly)
	return self:_Iterator(isReadOnly)
end

function DatabaseQuery.IteratorAndRelease(self, isReadOnly)
	return self:_Iterator(isReadOnly, true)
end

function DatabaseQuery._Iterator(self, isReadOnly, release)
	assert(self._rootClause and self._currentClause == self._rootClause, "Did not end sub-clause")
	-- try to find an index to use to optimize this query
	local indexField, indexValue = nil, nil
	for field in self._db:_UniquesIterator() do
		local value = self._rootClause:_GetIndexValue(field)
		if not indexField and value ~= nil then
			indexField = field
			indexValue = value
		end
	end
	if not indexField then
		for field in self._db:_IndexIterator() do
			local value = self._rootClause:_GetIndexValue(field)
			if not indexField and value ~= nil then
				indexField = field
				indexValue = value
			end
		end
	end
	-- get all the rows which match the query
	local result = nil
	local firstOrderBy = self._orderBy[1]
	local sortNeeded = firstOrderBy and true or false
	if indexField and self._db:_IsUnique(indexField) then
		-- we are looking for a unique row, so return an iterator which passes it through
		local context = TSMAPI_FOUR.Util.AcquireTempTable()
		context.row = self._db:GetUniqueRow(indexField, indexValue)
		context.selectList = #self._select > 0 and self._select or nil
		context.query = self
		context.releaseQuery = release and self or nil
		assert(self._iterReadOnlyState == nil)
		self._iterReadOnlyState = isReadOnly and true or false
		return private.PassThroughIterator, context, nil
	elseif indexField then
		-- we're querying on an index, so use that index to populate the result
		local isAscending = true
		if firstOrderBy and indexField == firstOrderBy then
			-- we're also ordering by this field so can skip the first OrderBy field
			sortNeeded = #self._orderBy > 1
			isAscending = self._orderByAscending[1]
		end
		if not sortNeeded and isReadOnly then
			local selectList = #self._select > 0 and self._select or nil
			assert(self._iterReadOnlyState == nil)
			self._iterReadOnlyState = isReadOnly and true or false
			return self._db:_RowByIndexIterator(indexField, indexValue, isAscending, self._distinct, selectList, release and self or nil, self)
		end
		result = TSMAPI_FOUR.Util.AcquireTempTable(result)
		assert(self._iterReadOnlyState == nil)
		self._iterReadOnlyState = isReadOnly and true or false
		for _, row in self._db:_RowByIndexIterator(indexField, indexValue, isAscending, self._distinct, nil, nil, self) do
			tinsert(result, row)
		end
	elseif firstOrderBy and self._db:_IsIndex(firstOrderBy) then
		-- we're ordering on an index, so use that index to iterate through all the rows in order to skip the first OrderBy field
		sortNeeded = #self._orderBy > 1
		local indexTable = self._db:_GetAllRowsByIndex(firstOrderBy)
		if not sortNeeded and isReadOnly then
			local context = TSMAPI_FOUR.Util.AcquireTempTable()
			context.query = self
			context.rows = indexTable
			context.ascending = self._orderByAscending[1]
			context.queryClause = self._rootClause
			context.selectList = #self._select > 0 and self._select or nil
			context.releaseQuery = release and self or nil
			if self._distinct then
				context.distinctField = self._distinct
				context.distinctUsed = TSMAPI_FOUR.Util.AcquireTempTable()
			end
			assert(self._iterReadOnlyState == nil)
			self._iterReadOnlyState = isReadOnly and true or false
			return private.RowByQueryIterator, context, context.ascending and 0 or #context.rows + 1
		end
		result = TSMAPI_FOUR.Util.AcquireTempTable(result)
		if self._orderByAscending[1] then
			for i = 1, #indexTable do
				local row = indexTable[i]
				if self._rootClause:_IsTrue(row) then
					tinsert(result, row)
				end
			end
		else
			for i = #indexTable, 1, -1 do
				local row = indexTable[i]
				if self._rootClause:_IsTrue(row) then
					tinsert(result, row)
				end
			end
		end
	elseif not sortNeeded and isReadOnly then
		local context = TSMAPI_FOUR.Util.AcquireTempTable()
		context.query = self
		context.rows = self._db:_GetRows()
		context.ascending = true
		context.queryClause = self._rootClause
		context.selectList = #self._select > 0 and self._select or nil
		context.releaseQuery = release and self or nil
		if self._distinct then
			context.distinctField = self._distinct
			context.distinctUsed = TSMAPI_FOUR.Util.AcquireTempTable()
		end
		assert(self._iterReadOnlyState == nil)
		self._iterReadOnlyState = isReadOnly and true or false
		return private.RowByQueryIterator, context, 0
	else
		result = TSMAPI_FOUR.Util.AcquireTempTable(result)
		-- no optimizations
		local distinctUsed = TSMAPI_FOUR.Util.AcquireTempTable()
		for _, row in ipairs(self._db:_GetRows()) do
			if self._rootClause:_IsTrue(row) then
				if self._distinct then
					local distinctValue = row:GetField(self._distinct)
					if not distinctUsed[distinctValue] then
						distinctUsed[distinctValue] = true
						tinsert(result, row)
					end
				else
					tinsert(result, row)
				end
			end
		end
		TSMAPI_FOUR.Util.ReleaseTempTable(distinctUsed)
	end
	assert(result)
	if sortNeeded then
		private.sortContext = self
		sort(result, private.DatabaseQuerySort)
		private.sortContext = nil
	end
	result._iterQuery = self
	result._iterSelectList = #self._select > 0 and self._select or nil
	result._iterReleaseQuery = release and self or nil
	assert(self._iterReadOnlyState == nil)
	self._iterReadOnlyState = isReadOnly and true or false
	return private.QueryResultIterator, result, 0
end

function DatabaseQuery.Count(self)
	local count = 0
	for _ in self:Iterator(true) do
		count = count + 1
	end
	return count
end

function DatabaseQuery.CountAndRelease(self)
	local count = self:Count()
	self:Release()
	return count
end

function DatabaseQuery.GetSingleResult(self)
	local result = nil
	for _, row in self:Iterator(true) do
		assert(not result)
		result = row
	end
	assert(result)
	return result
end

function DatabaseQuery.GetFirstResult(self)
	local result = nil
	for _, row in self:Iterator(true) do
		-- can't break out of the iterator so just continue looping and store the first one
		result = result or row
	end
	return result
end

function DatabaseQuery.GetSingleResultAndRelease(self)
	local result = self:GetSingleResult()
	self:Release()
	return result
end

function DatabaseQuery.GetFirstResultAndRelease(self)
	local result = self:GetFirstResult()
	self:Release()
	return result
end

function DatabaseQuery.Reset(self)
	self._distinct = nil
	wipe(self._select)
	self._rootClause:_Release()
	wipe(self._orderBy)
	wipe(self._orderByAscending)
	self._rootClause = private.NewDatabaseQueryClause(self)
		:And()
	self._currentClause = self._rootClause
end



-- ============================================================================
-- Database Class Definitions
-- ============================================================================

local Database = TSMAPI_FOUR.Class.DefineClass("Database")

function Database.__init(self, schema)
	self._schema = schema
	self._rows = {}
	self._queries = {}
	self._indexLists = {}
	self._uniques = {}
	self._queryUpdatesPaused = 0
	self._queuedQueryUpdate = false
	self._extensionCallbacks = {}

	for fieldName, fieldType in pairs(schema.fields) do
		assert(type(fieldName) == "string" and strsub(fieldName, 1, 1) ~= "_")
		assert(VALID_TYPES[fieldType])
	end
	if schema.fieldAttributes then
		for field, attributes in pairs(schema.fieldAttributes) do
			assert(schema.fields[field])
			-- make sure attributes is a list
			assert(TSMAPI_FOUR.Util.Count(attributes) == #attributes)
			for _, attribute in ipairs(attributes) do
				if attribute == "index" then
					self._indexLists[field] = {}
				elseif attribute == "unique" then
					self._uniques[field] = {}
				else
					error("Unknown field attribute: "..tostring(attribute))
				end
			end
		end
	end
end

function Database._GetFieldType(self, field)
	return self._schema.fields[field]
end

function Database._IsIndex(self, field)
	return self._indexLists[field] and true or false
end

function Database._IsUnique(self, field)
	return self._uniques[field] and true or false
end

function Database._IndexIterator(self)
	return TSMAPI_FOUR.Util.TableKeyIterator(self._indexLists)
end

function Database._UniquesIterator(self)
	return TSMAPI_FOUR.Util.TableKeyIterator(self._uniques)
end

function Database._GetRows(self)
	return self._rows
end

function Database._GetAllRowsByIndex(self, indexField)
	return self._indexLists[indexField]
end

function Database._RowByIndexIterator(self, indexField, indexValue, isAscending, distinctField, selectList, releaseQuery, query)
	assert(type(isAscending) == "boolean")
	local indexList = self._indexLists[indexField]
	-- figure out what row to start on
	local rowIndex = nil
	if isAscending then
		rowIndex = TSMAPI_FOUR.Util.BinarySearchGetFirstIndex(indexList, indexValue, indexField)
		rowIndex = rowIndex and (rowIndex - 1) or 0
	else
		rowIndex = TSMAPI_FOUR.Util.BinarySearchGetLastIndex(indexList, indexValue, indexField)
		rowIndex = rowIndex and (rowIndex + 1) or (#indexList + 1)
	end

	local context = TSMAPI_FOUR.Util.AcquireTempTable()
	context.query = query
	context.rows = indexList
	context.ascending = isAscending
	context.field = indexField
	context.value = indexValue
	context.selectList = selectList
	context.releaseQuery = releaseQuery
	if distinctField then
		context.distinctField = distinctField
		context.distinctUsed = TSMAPI_FOUR.Util.AcquireTempTable()
	end
	return private.RowByIndexIterator, context, rowIndex
end

function Database._RemoveQuery(self, queryToRemove)
	local queryIndex = nil
	for i, query in ipairs(self._queries) do
		if query == queryToRemove then
			assert(not queryIndex)
			queryIndex = i
		end
	end
	assert(queryIndex)
	tremove(self._queries, queryIndex)
end

function Database._UpdateQueries(self)
	for _, query in ipairs(self._queries) do
		assert(not query._iterReadOnlyState)
	end
	if self._queryUpdatesPaused > 0 then
		self._queuedQueryUpdate = true
	else
		self._queuedQueryUpdate = false
		for _, query in ipairs(self._queries) do
			query:_DataUpdated()
		end
	end
end

function Database._IndexListInsert(self, field, row)
	local indexList = self._indexLists[field]
	local insertIndex = TSMAPI_FOUR.Util.BinarySearchGetInsertIndex(indexList, row:GetField(field), field)
	tinsert(indexList, insertIndex, row)
end

function Database._InsertRow(self, row)
	tinsert(self._rows, row)
	for indexField in pairs(self._indexLists) do
		self:_IndexListInsert(indexField, row)
	end
	for uniqueField, values in pairs(self._uniques) do
		local rowValue = row:GetField(uniqueField)
		assert(not self._uniques[uniqueField][rowValue], "A row with this unique value already exists")
		self._uniques[uniqueField][rowValue] = row
	end
	self:_UpdateQueries()
end

function Database._UpdateRow(self, row, oldValues)
	for field, oldValue in pairs(oldValues) do
		local indexList = self._indexLists[field]
		local uniqueValues = self._uniques[field]
		if indexList then
			-- remove and re-add row to the index list since the index value changed
			TSMAPI_FOUR.Util.TableRemoveByValue(indexList, row)
			self:_IndexListInsert(field, row)
		elseif uniqueValues then
			assert(uniqueValues[oldValue] == row)
			uniqueValues[oldValue] = nil
			uniqueValues[row:GetField(field)] = row
		end
	end
	self:_UpdateQueries()
end

function Database._GetRowIndex(self, row)
	for i, existingRow in ipairs(self._rows) do
		if row == existingRow then
			return i
		end
	end
end

function Database.FieldIterator(self)
	return TSMAPI_FOUR.Util.TableKeyIterator(self._schema.fields)
end

function Database.ExtendSchema(self, schema, callback)
	assert(not schema.fieldAttributes, "Extending fieldAttributes is not currently supported")
	for fieldName, fieldType in pairs(schema.fields) do
		assert(type(fieldName) == "string" and strsub(fieldName, 1, 1) ~= "_")
		assert(VALID_TYPES[fieldType])
		assert(not self:_GetFieldType(fieldName))
		self._schema.fields[fieldName] = fieldType
	end
	tinsert(self._extensionCallbacks, callback)
	self:SetQueryUpdatesPaused(true)
	for _, row in ipairs(self:_GetRows()) do
		row:Save()
	end
	self:SetQueryUpdatesPaused(false)
end

function Database.NewRow(self)
	local row = nil
	if #private.recycledRows > 0 then
		row = tremove(private.recycledRows)
	else
		row = DatabaseRow()
	end
	row:_Acquire(self)
	return row
end

function Database.NewQuery(self)
	local query = nil
	if #private.recycledQueries > 0 then
		query = tremove(private.recycledQueries)
	else
		query = DatabaseQuery()
	end
	query:_Acquire(self)
	tinsert(self._queries, query)
	return query
end

function Database.DeleteRow(self, deleteRow)
	for _, indexList in pairs(self._indexLists) do
		TSMAPI_FOUR.Util.TableRemoveByValue(indexList, deleteRow)
	end
	for field, uniqueValues in pairs(self._uniques) do
		uniqueValues[deleteRow:GetField(field)] = nil
	end
	local rowIndex = self:_GetRowIndex(deleteRow)
	assert(rowIndex)
	tremove(self._rows, rowIndex)
	deleteRow:_Release()
	tinsert(private.recycledRows, deleteRow)
	self:_UpdateQueries()
end

function Database.Truncate(self)
	for _, row in ipairs(self._rows) do
		row:_Release()
		tinsert(private.recycledRows, row)
	end
	wipe(self._rows)
	for _, indexList in pairs(self._indexLists) do
		wipe(indexList)
	end
	for _, uniqueValues in pairs(self._uniques) do
		wipe(uniqueValues)
	end
	self:_UpdateQueries()
end

function Database.SetQueryUpdatesPaused(self, paused)
	self._queryUpdatesPaused = self._queryUpdatesPaused + (paused and 1 or -1)
	if self._queryUpdatesPaused == 0 and self._queuedQueryUpdate then
		self:_UpdateQueries()
	end
end

function Database.GetUniqueRow(self, field, value)
	local fieldType = self:_GetFieldType(field)
	if not fieldType then
		error(format("Field %s doesn't exist", tostring(field)), 3)
	elseif fieldType ~= type(value) then
		error(format("Field %s should be a %s, got %s", tostring(field), tostring(fieldType), type(value)), 3)
	end
	assert(self:_IsUnique(field))
	return self._uniques[field][value]
end



-- ============================================================================
-- TSMAPI Functions
-- ============================================================================

function TSMAPI_FOUR.Database.New(schema)
	return Database(schema)
end



-- ============================================================================
-- Private Helper Functions
-- ============================================================================

function private.DatabaseQuerySort(a, b)
	local self = private.sortContext
	for i = 1, #self._orderBy do
		local orderByField = self._orderBy[i]
		local aValue = a:GetField(orderByField)
		local bValue = b:GetField(orderByField)
		local fieldType = self._db:_GetFieldType(orderByField)
		if fieldType == "string" then
			aValue = strlower(aValue)
			bValue = strlower(bValue)
		elseif fieldType == "boolean" then
			aValue = aValue and 1 or 0
			bValue = bValue and 1 or 0
		end
		if aValue == bValue then
			-- continue looping
		elseif self._orderByAscending[i] then
			return aValue < bValue
		else
			return aValue > bValue
		end
	end
	-- make the sort stable
	return tostring(a) < tostring(b)
end

function private.NewDatabaseQueryClause(query, parent)
	local clause = nil
	if #private.recycledQueryClauses > 0 then
		clause = tremove(private.recycledQueryClauses)
	else
		clause = DatabaseQueryClause()
	end
	clause:_Acquire(query, parent)
	return clause
end

function private.RowByIndexIterator(context, index)
	while true do
		index = index + (context.ascending and 1 or -1)
		local row = context.rows[index]
		if not row or row[context.field] ~= context.value then
			assert(context.query._iterReadOnlyState ~= nil)
			context.query._iterReadOnlyState = nil
			if context.releaseQuery then
				context.releaseQuery:Release()
			end
			if context.distinctField then
				TSMAPI_FOUR.Util.ReleaseTempTable(context.distinctUsed)
			end
			TSMAPI_FOUR.Util.ReleaseTempTable(context)
			return
		elseif context.distinctField and context.distinctUsed[row[context.distinctField]] then
			-- skip this row
			row = nil
		end
		if row then
			if context.distinctField then
				context.distinctUsed[row[context.distinctField]] = true
			end
			if context.selectList then
				return index, row:GetFields(unpack(context.selectList))
			else
				return index, row
			end
		end
	end
end

function private.RowByQueryIterator(context, index)
	while true do
		index = index + (context.ascending and 1 or -1)
		local row = context.rows[index]
		if not row then
			assert(context.query._iterReadOnlyState ~= nil)
			context.query._iterReadOnlyState = nil
			if context.releaseQuery then
				context.releaseQuery:Release()
			end
			if context.distinctField then
				TSMAPI_FOUR.Util.ReleaseTempTable(context.distinctUsed)
			end
			TSMAPI_FOUR.Util.ReleaseTempTable(context)
			return
		elseif context.distinctField and context.distinctUsed[row:GetField(context.distinctField)] then
			-- continue looping
		elseif not context.queryClause:_IsTrue(row) then
			-- continue looping
		else
			if context.distinctField then
				context.distinctUsed[row:GetField(context.distinctField)] = true
			end
			if context.selectList then
				return index, row:GetFields(unpack(context.selectList))
			else
				return index, row
			end
		end
	end
end

function private.QueryRowsIteratorHelper(index, value, rootClause)
	if not rootClause:_IsTrue(value) then
		return
	end
	return index, value
end

function private.PassThroughIterator(context, index)
	if index or not context.row then
		assert(context.query._iterReadOnlyState ~= nil)
		context.query._iterReadOnlyState = nil
		if context.releaseQuery then
			context.releaseQuery:Release()
		end
		TSMAPI_FOUR.Util.ReleaseTempTable(context)
		return
	end
	if context.selectList then
		return 1, context.row:GetFields(unpack(context.selectList))
	else
		return 1, context.row
	end
end

function private.SelectIteator(index, value, arg)
	return index, value:GetFields(unpack(arg))
end

function private.QueryResultIterator(tbl, index)
	index = index + 1
	local row = tbl[index]
	if not row then
		assert(tbl._iterQuery._iterReadOnlyState ~= nil)
		tbl._iterQuery._iterReadOnlyState = nil
		if tbl._iterReleaseQuery then
			tbl._iterReleaseQuery:Release()
		end
		TSMAPI_FOUR.Util.ReleaseTempTable(tbl)
		return
	elseif tbl._iterSelectList then
		return index, row:GetFields(unpack(tbl._iterSelectList))
	else
		return index, row
	end
end
