文章目录[隐藏]

• 1.先验知识
• 2. VOC和COCO数据集： 
  • 2.1 VOC形式及其数据结构XML特点（好像可以使用py库中工具直接进行清洗）  
    •  2.1.1 VOC数据集的组织结构如下所示
    • 2.1.2 XML的操作 
    • 2.1.3 XML实例：country_data.xml
  • 2.2 coco数据集形式及其数据结构JOSN特点
    • 2.2.1 coco格式
• • 2.2.2 json数据结构及操作
• 3. 针对目标检测yolo5的一份数据清洗过程 
  • 3.1数据实例格式分析
  • Kvasir-SEG Dataset Details
  • 3.2 数据格式转换过程 
    • 3.2.2 xml标准化
    • 3.2.3 使用xml做成标准yolo标准数据集
• 4.参考文献：

#首先定义一个dicttoxml函数
#!/usr/bin/env python
# coding: utf-8

"""
Converts a Python dictionary or other native data type into a valid XML string. 

Supports item (`int`, `float`, `long`, `decimal.Decimal`, `bool`, `str`, `unicode`, `datetime`, `none` and other number-like objects) and collection (`list`, `set`, `tuple` and `dict`, as well as iterable and dict-like objects) data types, with arbitrary nesting for the collections. Items with a `datetime` type are converted to ISO format strings. Items with a `None` type become empty XML elements.

This module works with both Python 2 and 3.
"""

from __future__ import unicode_literals

__version__ = '1.7.4'
version = __version__

from random import randint
import collections
import numbers
import logging
from xml.dom.minidom import parseString


LOG = logging.getLogger("dicttoxml")

# python 3 doesn't have a unicode type
try:
    unicode
except:
    unicode = str

# python 3 doesn't have a long type
try:
    long
except:
    long = int


def set_debug(debug=True, filename='dicttoxml.log'):
    if debug:
        import datetime
        print('Debug mode is on. Events are logged at: %s' % (filename))
        logging.basicConfig(filename=filename, level=logging.INFO)
        LOG.info('\nLogging session starts: %s' % (
            str(datetime.datetime.today()))
        )
    else:
        logging.basicConfig(level=logging.WARNING)
        print('Debug mode is off.')


def unicode_me(something):
    """Converts strings with non-ASCII characters to unicode for LOG. 
    Python 3 doesn't have a `unicode()` function, so `unicode()` is an alias 
    for `str()`, but `str()` doesn't take a second argument, hence this kludge.
    """
    try:
        return unicode(something, 'utf-8')
    except:
        return unicode(something)
        

ids = [] # initialize list of unique ids

def make_id(element, start=100000, end=999999):
    """Returns a random integer"""
    return '%s_%s' % (element, randint(start, end))


def get_unique_id(element):
    """Returns a unique id for a given element"""
    this_id = make_id(element)
    dup = True
    while dup:
        if this_id not in ids:
            dup = False
            ids.append(this_id)
        else:
            this_id = make_id(element)
    return ids[-1]


def get_xml_type(val):
    """Returns the data type for the xml type attribute"""
    if type(val).__name__ in ('str', 'unicode'):
        return 'str'
    if type(val).__name__ in ('int', 'long'):
        return 'int'
    if type(val).__name__ == 'float':
        return 'float'
    if type(val).__name__ == 'bool':
        return 'bool'
    if isinstance(val, numbers.Number):
        return 'number'
    if type(val).__name__ == 'NoneType':
        return 'null'
    if isinstance(val, dict):
        return 'dict'
    if isinstance(val, collections.Iterable):
        return 'list'
    return type(val).__name__


def escape_xml(s):
    if type(s) in (str, unicode):
        s = unicode_me(s) # avoid UnicodeDecodeError
        s = s.replace('&', '&')
        s = s.replace('"', '"')
        s = s.replace('\'', ''')
        s = s.replace('<', '&lt;')
        s = s.replace('>', '&gt;')
    return s


def make_attrstring(attr):
    """Returns an attribute string in the form key="val" """
    attrstring = ' '.join(['%s="%s"' % (k, v) for k, v in attr.items()])
    return '%s%s' % (' ' if attrstring != '' else '', attrstring)


def key_is_valid_xml(key):
    """Checks that a key is a valid XML name"""
    LOG.info('Inside key_is_valid_xml(). Testing "%s"' % (unicode_me(key)))
    test_xml = '<?xml version="1.0" encoding="UTF-8" ?><%s>foo</%s>' % (key, key)
    try:
        parseString(test_xml)
        return True
    except Exception: # minidom does not implement exceptions well
        return False


def make_valid_xml_name(key, attr):
    """Tests an XML name and fixes it if invalid"""
    LOG.info('Inside make_valid_xml_name(). Testing key "%s" with attr "%s"' % (
        unicode_me(key), unicode_me(attr))
    )
    key = escape_xml(key)
    attr = escape_xml(attr)
    
    # pass through if key is already valid
    if key_is_valid_xml(key):
        return key, attr
        
    # prepend a lowercase n if the key is numeric
    if key.isdigit():
        return 'n%s' % (key), attr
        
    # replace spaces with underscores if that fixes the problem
    if key_is_valid_xml(key.replace(' ', '_')):
        return key.replace(' ', '_'), attr
        
    # key is still invalid - move it into a name attribute
    attr['name'] = key
    key = 'key'
    return key, attr


def wrap_cdata(s):
    """Wraps a string into CDATA sections"""
    s = unicode_me(s).replace(']]>', ']]]]><![CDATA[>')
    return '<![CDATA[' + s + ']]>'


def default_item_func(parent):
    return 'item'


def convert(obj, ids, attr_type, item_func, cdata, parent='root'):
    """Routes the elements of an object to the right function to convert them 
    based on their data type"""
    
    LOG.info('Inside convert(). obj type is: "%s", obj="%s"' % (type(obj).__name__, unicode_me(obj)))
    
    item_name = item_func(parent)
    
    if isinstance(obj, numbers.Number) or type(obj) in (str, unicode):
        return convert_kv(item_name, obj, attr_type, cdata)
        
    if hasattr(obj, 'isoformat'):
        return convert_kv(item_name, obj.isoformat(), attr_type, cdata)
        
    if type(obj) == bool:
        return convert_bool(item_name, obj, attr_type, cdata)
        
    if obj is None:
        return convert_none(item_name, '', attr_type, cdata)
        
    if isinstance(obj, dict):
        return convert_dict(obj, ids, parent, attr_type, item_func, cdata)
        
    if isinstance(obj, collections.Iterable):
        return convert_list(obj, ids, parent, attr_type, item_func, cdata)
        
    raise TypeError('Unsupported data type: %s (%s)' % (obj, type(obj).__name__))


def convert_dict(obj, ids, parent, attr_type, item_func, cdata):
    """Converts a dict into an XML string."""
    LOG.info('Inside convert_dict(): obj type is: "%s", obj="%s"' % (
        type(obj).__name__, unicode_me(obj))
    )
    output = []
    addline = output.append
    
    item_name = item_func(parent)
    
    for key, val in obj.items():
        LOG.info('Looping inside convert_dict(): key="%s", val="%s", type(val)="%s"' % (
            unicode_me(key), unicode_me(val), type(val).__name__)
        )

        attr = {} if not ids else {'id': '%s' % (get_unique_id(parent)) }

        key, attr = make_valid_xml_name(key, attr)

        if isinstance(val, numbers.Number) or type(val) in (str, unicode):
            addline(convert_kv(key, val, attr_type, attr, cdata))

        elif hasattr(val, 'isoformat'): # datetime
            addline(convert_kv(key, val.isoformat(), attr_type, attr, cdata))

        elif type(val) == bool:
            addline(convert_bool(key, val, attr_type, attr, cdata))

        elif isinstance(val, dict):
            if attr_type:
                attr['type'] = get_xml_type(val)
            addline('<%s%s>%s</%s>' % (
                key, make_attrstring(attr), 
                convert_dict(val, ids, key, attr_type, item_func, cdata), 
                key
                )
            )

        elif isinstance(val, collections.Iterable):
            if attr_type:
                attr['type'] = get_xml_type(val)
            addline('<%s%s>%s</%s>' % (
                key, 
                make_attrstring(attr), 
                convert_list(val, ids, key, attr_type, item_func, cdata), 
                key
                )
            )

        elif val is None:
            addline(convert_none(key, val, attr_type, attr, cdata))

        else:
            raise TypeError('Unsupported data type: %s (%s)' % (
                val, type(val).__name__)
            )

    return ''.join(output)


def convert_list(items, ids, parent, attr_type, item_func, cdata):
    """Converts a list into an XML string."""
    LOG.info('Inside convert_list()')
    output = []
    addline = output.append

    item_name = item_func(parent)

    if ids:
        this_id = get_unique_id(parent)

    for i, item in enumerate(items):
        LOG.info('Looping inside convert_list(): item="%s", item_name="%s", type="%s"' % (
            unicode_me(item), item_name, type(item).__name__)
        )
        attr = {} if not ids else { 'id': '%s_%s' % (this_id, i+1) }
        if isinstance(item, numbers.Number) or type(item) in (str, unicode):
            addline(convert_kv(item_name, item, attr_type, attr, cdata))
            
        elif hasattr(item, 'isoformat'): # datetime
            addline(convert_kv(item_name, item.isoformat(), attr_type, attr, cdata))
            
        elif type(item) == bool:
            addline(convert_bool(item_name, item, attr_type, attr, cdata))
            
        elif isinstance(item, dict):
            if not attr_type:
                addline('<%s>%s</%s>' % (
                    item_name, 
                    convert_dict(item, ids, parent, attr_type, item_func, cdata),
                    item_name, 
                    )
                )
            else:
                addline('<%s type="dict">%s</%s>' % (
                    item_name, 
                    convert_dict(item, ids, parent, attr_type, item_func, cdata),
                    item_name, 
                    )
                )

        elif isinstance(item, collections.Iterable):
            if not attr_type:
                addline('<%s %s>%s</%s>' % (
                    item_name, make_attrstring(attr), 
                    convert_list(item, ids, item_name, attr_type, item_func, cdata),
                    item_name, 
                    )
                )
            else:
                addline('<%s type="list"%s>%s</%s>' % (
                    item_name, make_attrstring(attr), 
                    convert_list(item, ids, item_name, attr_type, item_func, cdata),
                    item_name, 
                    )
                )
                
        elif item is None:
            addline(convert_none(item_name, None, attr_type, attr, cdata))
            
        else:
            raise TypeError('Unsupported data type: %s (%s)' % (
                item, type(item).__name__)
            )
    return ''.join(output)


def convert_kv(key, val, attr_type, attr={}, cdata=False):
    """Converts a number or string into an XML element"""
    LOG.info('Inside convert_kv(): key="%s", val="%s", type(val) is: "%s"' % (
        unicode_me(key), unicode_me(val), type(val).__name__)
    )

    key, attr = make_valid_xml_name(key, attr)

    if attr_type:
        attr['type'] = get_xml_type(val)
    attrstring = make_attrstring(attr)
    return '<%s%s>%s</%s>' % (
        key, attrstring, 
        wrap_cdata(val) if cdata == True else escape_xml(val),
        key
    )


def convert_bool(key, val, attr_type, attr={}, cdata=False):
    """Converts a boolean into an XML element"""
    LOG.info('Inside convert_bool(): key="%s", val="%s", type(val) is: "%s"' % (
        unicode_me(key), unicode_me(val), type(val).__name__)
    )

    key, attr = make_valid_xml_name(key, attr)

    if attr_type:
        attr['type'] = get_xml_type(val)
    attrstring = make_attrstring(attr)
    return '<%s%s>%s</%s>' % (key, attrstring, unicode(val).lower(), key)


def convert_none(key, val, attr_type, attr={}, cdata=False):
    """Converts a null value into an XML element"""
    LOG.info('Inside convert_none(): key="%s"' % (unicode_me(key)))

    key, attr = make_valid_xml_name(key, attr)

    if attr_type:
        attr['type'] = get_xml_type(val)
    attrstring = make_attrstring(attr)
    return '<%s%s></%s>' % (key, attrstring, key)


def dicttoxml(obj, root=True, custom_root='root', ids=False, attr_type=True,
    item_func=default_item_func, cdata=False):
    """Converts a python object into XML.
    Arguments:
    - root specifies whether the output is wrapped in an XML root element
      Default is True
    - custom_root allows you to specify a custom root element.
      Default is 'root'
    - ids specifies whether elements get unique ids.
      Default is False
    - attr_type specifies whether elements get a data type attribute.
      Default is True
    - item_func specifies what function should generate the element name for
      items in a list. 
      Default is 'item'
    - cdata specifies whether string values should be wrapped in CDATA sections.
      Default is False
    """
    LOG.info('Inside dicttoxml(): type(obj) is: "%s", obj="%s"' % (type(obj).__name__, unicode_me(obj)))
    output = []
    addline = output.append
    if root == True:
        addline('<?xml version="1.0" encoding="UTF-8" ?>')
        addline('<%s>%s</%s>' % (
        custom_root, 
        convert(obj, ids, attr_type, item_func, cdata, parent=custom_root), 
        custom_root,
        )
    )
    else:
        addline(convert(obj, ids, attr_type, item_func, cdata, parent=''))
    return ''.join(output).encode('utf-8')

import json
import urllib
import dicttoxml
#page = urllib.urlopen(r'D:\pythonProject1\json\annotation.json')
#print(page)
def load_josn(path):
    lines = []
    with open(path) as f:
        for row in f.readlines():
            if row.strip().startswith("//"):
                continue
            lines.append(row)
    return json.loads("\n".join(lines))

if __name__ == "__main__":


    obj = load_josn(r"D:\pythonProject1\json\annotation.json")
    #print(obj)
    xml = dicttoxml.dicttoxml(obj)
    print(xml)

<?xml version="1.0" encoding="UTF-8" ?><root>
<cju2ouil2mssu0993hvxsed6d type="dict"><width type="int">605</width>
<bbox type="list"><item type="dict"><xmin type="int">246</xmin><ymin type="int">243</ymin><ymax type="int">360</ymax><xmax type="int">361</xmax><label type="str">polyp</label></item></bbox><height type="int">511</height>

</cju2ouil2mssu0993hvxsed6d><cju5fs6j6d8350801vglraq4u type="dict"><width type="int">626</width><bbox type="list"><item type="dict"><xmin type="int">72</xmin><ymin type="int">146</ymin><ymax type="int">450</ymax><xmax type="int">415</xmax><label type="str">polyp</label></item></bbox><height type="int">547</height>

</cju5fs6j6d8350801vglraq4u><cju34sh43d8zm08019xbwhc0o type="dict"><width type="int">620</width><bbox type="list"><item type="dict"><xmin type="int">105</xmin><ymin type="int">206</ymin><ymax type="int">492</ymax><xmax type="int">332</xmax><label type="str">polyp</label></item></bbox><height type="int">530</height>

</cju34sh43d8zm08019xbwhc0o><cju422cm8lfxn0818ojicxejb type="dict"><width type="int">571</width><bbox type="list"><item type="dict"><xmin type="int">147</xmin><ymin type="int">346</ymin><ymax type="int">531</ymax><xmax type="int">376</xmax><label type="str">polyp</label></item></bbox><height type="int">531</height>

#encoding: utf-8
import copy
#from lxml import etree
#from lxml.etree import Element as ET
#from lxml.etree import Element, SubElement, tostring, ElementTree
import xml.etree.ElementTree as ET
import cv2
import os


# 模板xlm文件的存储路径
template_file = r'D:\pythonProject1\model.xml'
path = r'D:/pythonProject1/images/'

# TXT文件数据的原始格式
train_files = 'cju0roawvklrq0799vmjorwfv.jpg id12 1.3418148e+03, 6.2916492e+02, 1.4483580e+03, 9.2253162e+02, 6.3271374e-01'
trainFile = train_files.split()  # trainFile存放全量的原始数据
#print('原始数据集格式：{}'.format(trainFile))

file_name = trainFile[0]
print(file_name)


# 定义新的xlm文件的详细指标数据
label = trainFile[1]
xmin = trainFile[2]
ymin = trainFile[3]
xmax = trainFile[4]
ymax = trainFile[5]

############# 读取模板xlm文件——用于存放TXT文件内容:
tree = ET.parse(template_file)  # 调用parse()方法，返回解析树
root = tree.getroot()  # 获取根节点

##########修改新的xlm文件的详细指标数据
# folder
root.find('folder').text = 'new_folders'
# 修改魔板xlm文件中的内容为目标结果
root.find('filename').text = file_name  # 2.Element.find() :找到第一个带有特定标签的子元素。
# # path
root.find('path').text = path + file_name
# 查看部分修改结果

print(root.find('filename').text)  # 第一层下每一项内容
print(root.find('path').text)  # 第一层下每一项内容
print(root.find('size').find('height').text)  # 查看第二层下每一项内容
print(root.find('object').find('bndbox').find('xmin').text)  # 查看第三层下每一项内容


# size
imgpath = os.path.join(path, file_name)
sz = root.find('size')
im = cv2.imread(path + file_name)  # 读取图片信息

sz.find('height').text = str(im.shape[0])
sz.find('width').text = str(im.shape[1])
sz.find('depth').text = str(im.shape[2])

print('iamge height:', im.shape[0])
print('iamge width:', im.shape[1])
print('iamge depth:', im.shape[2])


# object
obj = root.find('object')

obj.find('name').text = label
bb = obj.find('bndbox')
bb.find('xmin').text = xmin
bb.find('ymin').text = ymin
bb.find('xmax').text = xmax
bb.find('ymax').text = ymax

########## 校验修改后的root是否为新数据
root.find('object').find('bndbox').find('ymax').text

########## 保存新生成的xlm数据文件
tree = ET.ElementTree(root)
tree.write(r"D:\pythonProject1\99.xml")