overspend1/PathOfBuilding
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1952 from Nostrademous/Dat64

Browse Source 
Dat64 GGPK Viewer
This commit is contained in:
Trevor Lund
2021-02-06 15:03:47 -06:00
committed by GitHub
parent f9a75aa821 06bc64abf3
commit bbfb22b41c
4 changed files with 388 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

289
Export/Classes/Dat64File.lua Normal file
View File

@@ -0,0 +1,289 @@
-- Dat View
--
-- Class: Dat64 File
-- Dat64 File
--
local ipairs = ipairs
local t_insert = table.insert
local m_min = math.min
local dataTypes = {
Bool = {
size = 1,
read = function(b, o, d)
return b:byte(o) == 1
end,
},
Int = {
size = 4,
read = function(b, o, d)
if o > #b - 3 then return -1337 end
return bytesToInt(b, o)
end,
},
UInt = {
size = 4,
read = function(b, o, d)
if o > #b - 3 then return 1337 end
return bytesToUInt(b, o)
end,
},
Interval = {
size = 8,
read = function(b, o, d)
if o > #b - 7 then return { 1337, 1337 } end
return { bytesToInt(b, o), bytesToInt(b, o + 4) }
end,
},
Float = {
size = 4,
read = function(b, o, d)
if o > #b - 3 then return -1337 end
return bytesToFloat(b, o)
end,
},
String = {
size = 8,
read = function(b, o, d)
if o > #b - 3 then return "<no offset>" end
local stro = bytesToULong(b, o)
if stro > #b - 3 then return "<bad offset>" end
return convertUTF16to8(b, d + stro)
end,
},
Enum = {
size = 4,
ref = true,
read = function(b, o, d)
if o > #b - 3 then return 1337 end
return bytesToUInt(b, o)
end,
},
Key = {
size = 16,
ref = true,
read = function(b, o, d)
if o > #b - 7 then return 1337 end
return bytesToULong(b, o)
end,
},
}
local Dat64FileClass = newClass("Dat64File", function(self, name, raw)
self.name = name
self.raw = raw
if not main.datSpecs[self.name] then
main.datSpecs[self.name] = { }
end
self.spec = main.datSpecs[self.name]
self.cols = { }
self.colMap = { }
self.indexes = { }
local colMeta = { __index = function(t, key)
local colIndex = self.colMap[key]
if not colIndex then
error("Unknown key "..key.." for "..self.name..".dat64")
end
t[key] = self:ReadCell(t._rowIndex, colIndex)
return rawget(t, key)
end }
self.rowCache = setmetatable({ }, { __index = function(t, rowIndex)
if rowIndex < 1 or rowIndex > self.rowCount then
return
end
t[rowIndex] = setmetatable({ _rowIndex = rowIndex }, colMeta)
return t[rowIndex]
end })
self.rows = { }
self.rowCount = bytesToUInt(self.raw)
self.dataOffset = self.raw:find("\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB", 5, true) or (#self.raw + 1)
self.rowSize = (self.dataOffset - 5) / self.rowCount
for i = 1, self.rowCount do
self.rows[i] = 5 + (i-1) * self.rowSize
end
--ConPrintf("Loaded '%s': %d Rows at %d Bytes", self.name, self.rowCount, self.rowSize)
self:OnSpecChanged()
end)
function Dat64FileClass:OnSpecChanged()
wipeTable(self.cols)
wipeTable(self.colMap)
wipeTable(self.indexes)
wipeTable(self.rowCache)
local offset = 0
for i, specCol in ipairs(self.spec) do
local dataType = dataTypes[specCol.type]
local size = specCol.list and 16 or dataType.size
self.cols[i] = {
size = size,
offset = offset,
isRef = dataType.ref,
}
offset = offset + size
if #specCol.name > 0 then
self.colMap[specCol.name] = i
end
end
self.specSize = offset
self.cols[#self.spec + 1] = {
size = self.rowSize - offset,
offset = offset,
}
end
function Dat64FileClass:GetRow(key, value)
local keyIndex = self.colMap[key]
if not keyIndex then
error("Unknown key "..key.." for "..self.name..".dat64")
end
if not self.indexes[key] then
self.indexes[key] = { }
end
for rowIndex = 1, self.rowCount do
if not self.indexes[key][rowIndex] then
self.indexes[key][rowIndex] = self:ReadCell(rowIndex, keyIndex)
end
if self.indexes[key][rowIndex] == value then
return self.rowCache[rowIndex]
end
end
end
function Dat64FileClass:GetRowByIndex(rowIndex)
return self.rowCache[rowIndex]
end
function Dat64FileClass:Rows()
local i = 0
return function()
i = i + 1
if i <= self.rowCount then
return self:GetRowByIndex(i)
end
end
end
function Dat64FileClass:GetRowList(key, value, match)
local keyIndex = self.colMap[key]
if not keyIndex then
error("Unknown key "..key.." for "..self.name..".dat64")
end
local isList = self.spec[keyIndex].list
if not self.indexes[key] then
self.indexes[key] = { }
end
local out = { }
for rowIndex = 1, self.rowCount do
if not self.indexes[key][rowIndex] then
self.indexes[key][rowIndex] = self:ReadCell(rowIndex, keyIndex)
end
local index = self.indexes[key][rowIndex]
if isList then
for _, indexVal in ipairs(index) do
if (match and indexVal:match(value)) or (not match and indexVal == value) then
t_insert(out, self.rowCache[rowIndex])
break
end
end
else
if (match and index:match(value)) or (not match and index == value) then
t_insert(out, self.rowCache[rowIndex])
end
end
end
return out
end
function Dat64FileClass:ReadCell(rowIndex, colIndex)
local spec = self.spec[colIndex]
local col = self.cols[colIndex]
local base = self.rows[rowIndex] + col.offset
if spec.list then
local dataType = dataTypes[spec.type]
local count = bytesToULong(self.raw, base)
local offset = bytesToULong(self.raw, base + 8) + self.dataOffset
local out = { }
for i = 1, m_min(count, 1000) do
out[i] = self:ReadValue(spec, offset)
offset = offset + dataType.size
end
return out
else
return self:ReadValue(spec, base)
end
end
function Dat64FileClass:ReadValue(spec, offset)
local dataType = dataTypes[spec.type]
local val = dataType.read(self.raw, offset, self.dataOffset)
if not dataType.ref then
return val
end
if val == 0xFEFEFEFE or val == 0xFEFEFEFEFEFEFEFE then
return
end
local other = main.datFileByName[spec.refTo]
if not other then
return
end
if spec.type == "Enum" and spec.refTo ~= self.name then
return val
end
return other.rowCache[val + 1]
end
function Dat64FileClass:ReadCellText(rowIndex, colIndex)
local spec = self.spec[colIndex]
local col = self.cols[colIndex]
local base = self.rows[rowIndex] + col.offset
if spec.list then
local dataType = dataTypes[spec.type]
local count = bytesToULong(self.raw, base)
local offset = bytesToULong(self.raw, base + 8) + self.dataOffset
local out = { }
for i = 1, m_min(count, 1000) do
out[i] = self:ReadValueText(spec, offset)
offset = offset + dataType.size
end
return out
else
return self:ReadValueText(spec, base)
end
end
function Dat64FileClass:ReadValueText(spec, offset)
local dataType = dataTypes[spec.type]
local val = dataType.read(self.raw, offset, self.dataOffset)
if dataType.ref then
if val == 0xFEFEFEFE or val == 0xFEFEFEFEFEFEFEFE then
return ""
end
local other = main.datFileByName[spec.refTo]
if other then
local otherRow = other.rows[val + ((spec.type == "Enum" and spec.refTo ~= self.name) and 0 or 1)]
if not otherRow then
return "<bad ref #"..val..">"
end
if other.spec[1] then
return other:ReadValueText(other.spec[1], otherRow)
end
end
end
if spec.type == "Interval" then
return val[1] == val[2] and val[1] or (val[1] .. " to " .. val[2])
else
return val
end
end
function Dat64FileClass:ReadCellRaw(rowIndex, colIndex)
local col = self.cols[colIndex]
local base = self.rows[rowIndex] + col.offset
return self.raw:byte(base, base + col.size - 1)
end

26
Export/Classes/GGPKData.lua
View File

@@ -39,7 +39,12 @@ local GGPKClass = newClass("GGPKData", function(self, path)
self.txt = { }
self:ExtractFiles()
self:AddDatFiles()
if USE_DAT64 then
self:AddDat64Files()
else
self:AddDatFiles()
end
end)
function GGPKClass:ExtractFiles()
@@ -47,7 +52,11 @@ function GGPKClass:ExtractFiles()
local fileList = ''
for _, fname in ipairs(datList) do
fileList = fileList .. '"' .. fname .. '" '
if USE_DAT64 then
fileList = fileList .. '"' .. fname .. '64" '
else
fileList = fileList .. '"' .. fname .. '" '
end
end
for _, fname in ipairs(txtList) do
fileList = fileList .. '"' .. fname .. '" '
@@ -74,6 +83,19 @@ function GGPKClass:AddDatFiles()
end
end
function GGPKClass:AddDat64Files()
local datFiles = scanDir(self.oozPath .. "Data\\", '%w+%.dat64$')
for _, f in ipairs(datFiles) do
local record = { }
record.name = f
local rawFile = io.open(self.oozPath .. "Data\\" .. f, 'rb')
record.data = rawFile:read("*all")
rawFile:close()
--ConPrintf("FILENAME: %s", fname)
t_insert(self.dat, record)
end
end
function GGPKClass:GetNeededFiles()
local datFiles = {
"Data/Stats.dat",

44
Export/Main.lua
View File

@@ -3,6 +3,9 @@
-- Module: Main
-- Main module of program.
--
USE_DAT64 = false
local ipairs = ipairs
local t_insert = table.insert
local t_remove = table.remove
@@ -46,6 +49,7 @@ local ourClassList = {
"RowListControl",
"SpecColListControl",
"DatFile",
"Dat64File",
"GGPKData",
}
for _, className in ipairs(classList) do
@@ -68,7 +72,12 @@ function main:Init()
self.datFileByName = { }
self:LoadSettings()
self:LoadDatFiles()
if USE_DAT64 then
self:LoadDat64Files()
else
self:LoadDatFiles()
end
self.scriptList = { }
local handle = NewFileSearch("Scripts/*.lua")
@@ -140,7 +149,11 @@ function main:Init()
self.controls.datSource = new("EditControl", nil, 10, 30, 250, 18, self.datSource) {
enterFunc = function(buf)
self.datSource = buf
self:LoadDatFiles()
if USE_DAT64 then
self:LoadDat64Files()
else
self:LoadDatFiles()
end
end
}
@@ -326,6 +339,33 @@ function main:LoadDatFiles()
end
end
function main:LoadDat64Files()
wipeTable(self.datFileList)
wipeTable(self.datFileByName)
self:SetCurrentDat()
self.ggpk = nil
if not self.datSource then
return
elseif self.datSource:match("%.ggpk") or self.datSource:match("steamapps[/\\].+[/\\]Path of Exile") then
local now = GetTime()
self.ggpk = new("GGPKData", self.datSource)
ConPrintf("GGPK: %d ms", GetTime() - now)
now = GetTime()
for i, record in ipairs(self.ggpk.dat) do
if i == 1 then
ConPrintf("DAT64 find: %d ms", GetTime() - now)
now = GetTime()
end
local datFile = new("Dat64File", record.name:gsub("%.dat64$",""), record.data)
t_insert(self.datFileList, datFile)
self.datFileByName[datFile.name] = datFile
end
ConPrintf("DAT64 read: %d ms", GetTime() - now)
end
end
function main:SetCurrentDat(datFile)
self.curDatFile = datFile
if datFile then

66
Export/spec.lua
View File

@@ -1929,91 +1929,91 @@ return {
},
[2]={
list=false,
name="",
name="Unknown1",
refTo="",
type="Key",
width=50
},
[3]={
list=false,
name="",
name="Unknown2",
refTo="",
type="Key",
width=50
},
[4]={
list=false,
name="",
name="Unknown3",
refTo="",
type="Key",
width=50
},
[5]={
list=false,
name="",
name="Unknown4",
refTo="",
type="Key",
width=50
},
[6]={
list=false,
name="",
name="Unknown5",
refTo="",
type="Key",
width=50
},
[7]={
list=false,
name="",
name="Unknown6",
refTo="",
type="Key",
width=50
},
[8]={
list=false,
name="",
name="Unknown7",
refTo="",
type="Key",
width=50
},
[9]={
list=false,
name="",
name="Unknown8",
refTo="",
type="Key",
width=50
},
[10]={
list=false,
name="",
name="Unknown9",
refTo="",
type="Key",
width=50
},
[11]={
list=false,
name="",
name="Unknown10",
refTo="",
type="Key",
width=50
},
[12]={
list=false,
name="",
name="Unknown11",
refTo="",
type="Key",
width=50
},
[13]={
list=false,
name="",
name="WandsMod",
refTo="Mods",
type="Key",
width=160
},
[14]={
list=false,
name="",
name="BowMods",
refTo="Mods",
type="Key",
width=150
@@ -2027,101 +2027,101 @@ return {
},
[16]={
list=false,
name="",
name="AmuletMods",
refTo="Mods",
type="Key",
width=150
},
[17]={
list=false,
name="",
name="RingMods",
refTo="Mods",
type="Key",
width=150
},
[18]={
list=false,
name="",
name="BeltMods",
refTo="Mods",
type="Key",
width=150
},
[19]={
list=false,
name="",
name="GlovesMods",
refTo="Mods",
type="Key",
width=150
},
[20]={
list=false,
name="",
name="BootsMods",
refTo="Mods",
type="Key",
width=150
},
[21]={
list=false,
name="",
name="BodyArmourMods",
refTo="Mods",
type="Key",
width=150
},
[22]={
list=false,
name="",
name="HelmetMods",
refTo="Mods",
type="Key",
width=150
},
[23]={
list=false,
name="",
name="ShieldMods",
refTo="Mods",
type="Key",
width=150
},
[24]={
list=false,
name="",
name="Unknown23",
refTo="",
type="UInt",
width=100
},
[25]={
list=false,
name="",
name="DropLevelMinimum",
refTo="",
type="Interval",
width=80
type="Int",
width=100
},
[26]={
list=false,
name="",
name="DropLevelMaximum",
refTo="",
type="Int",
width=50
},
[27]={
list=false,
name="",
refTo="",
type="Int",
width=60
list=true,
name="MonterMods",
refTo="Mods",
type="Key",
width=300
},
[28]={
list=false,
name="Type",
refTo="EssenceType",
type="Key",
width=70
width=150
},
[29]={
list=false,
name="Tier",
refTo="",
type="Int",
width=50
width=80
},
[30]={
list=false,