将XML导入PostgreSQL数据库或转换XML -> CSV -> PostgreSQL

Question

我有XML文件

<Cluster clsId="UNIPR_NIRI_PARDP" semType="geneProt"> <Entry entryId="UNIPR_NIRI_PARDP_1" baseForm="Protein nirI" type="PREFERRED">

<Variant WRITTENFORM="FMN-binding domain protein" type="orthographic"/> <Variant WRITTENFORM="FMN-binding domain-containing protein" type="orthographic"/> <Variant WRITTENFORM="unknown" type="orthographic"/> <Variant WRITTENFORM="FMN-binding" type="orthographic"/> <Variant WRITTENFORM="Pden_2486" type="orthographic"/> <Variant WRITTENFORM="nirI" type="orthographic"/> <SourceDC sourceName="BioThesaurus" sourceId="Q51699"/> <PosDC posName="POS" pos="N"/> <DC att="uniprot_ac" val="Q51699"/> <DC att="speciesNameNCBI" val="318586"/>

</Entry> </Cluster>

我需要将此内容导入postgresql。请帮助我在这方面无论是直接的程序或转换XML到csv到PostgreSQL。

我需要具有如下列的表

clsid、entryid _dc、semType、baseForm、变量(书写形式)、变量(类型)、dc(属性)、dc(val)

提前谢谢你。

Answer 1

我使用Ruby在包noko-giri和open-uri的帮助下做到了这一点。因为我的输入文件太大了。许多解析器都失败了，而noko-giri在这方面提供了帮助。

我提供了三列答案，baseForm-variant(writtenform)-dc(val)。这个信息可能是问题的一个清晰的信息。

require 'nokogiri'
require 'open-uri'

doc = Nokogiri::XML(File.open("xai"))
ent = doc.xpath("//Entry")

value = String.new
ent.each do |e| 
    d = e.xpath("DC")   
    d.each do |f|       
        if f.attributes["att"].to_s =~ /uniprot_ac/
            value = f.attributes["val"].to_s
        end
    end
    f = e.xpath("Variant")  
    f.each do |g|
        puts "#{e.attributes["baseForm"].to_s}\t" + "#{g.attributes["WRITTENFORM"].to_s}\t" + "#{value}"
    end 
end

Answer 2

首先，解析您的xml文件以获得一个包含您所需的所有信息的文件。

例如，如果您只想拥有一个具有clsid、entryid、semType、baseForm、variant(写入形式)、variant(类型)、dc(att)、dc(val)属性的表，那么您只需要一个具有这些属性(用某些字符分隔)的文件。文件中的每一行都对应于表中的每一行。

接下来，在Postgresql中创建表模式。然后使用Postgresql的COPY命令，该命令将所有数据从文件复制到表中。

注意，如果你的xml文件很大，你应该使用基于事件的解析器。类似于SAX，例如Java语言中的StAX。

编辑核心*注意*：使用的库: stax2-api-3.1.1.jar，woodstox--asl-4.1.1jar这里是代码(希望它能做你需要的事情，如果不能，我相信它能帮你入门)：

/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package test;

import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.net.MalformedURLException;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import java.util.ArrayList;
import org.codehaus.stax2.XMLInputFactory2;
import org.codehaus.stax2.XMLStreamReader2;

public class Main {

    /**
     * @param args the command line arguments
     */

    /*
     * dc(att), dc(val)
     */
    @SuppressWarnings("CallToThreadDumpStack")
    public static void main(String[] args) throws MalformedURLException, IOException, XMLStreamException {
        FileInputStream fstream = new FileInputStream(args[0]);
        Reader in = new InputStreamReader(fstream, "UTF-8");
        XMLInputFactory2 factory = (XMLInputFactory2) XMLInputFactory.newInstance();
        XMLStreamReader2 parser = (XMLStreamReader2) factory.createXMLStreamReader(in);

        FileOutputStream outStream = new FileOutputStream("/home/aseke/Desktop/out.txt");
        BufferedWriter out = new BufferedWriter(new OutputStreamWriter(outStream, "UTF-8"));


        boolean isCluster = false;
        ArrayList<String> dc = new ArrayList<String>();
        ArrayList<String> variants = new ArrayList<String>();

        /* You actually do not need all of these variables, it's just for clarity */
        String clsID = null;
        String semType = null;
        String varWritten = null;
        String varType = null;
        String entryID = null;
        String baseForm = null;
        String dcAtt = null;
        String dcVal = null;
        String s = null;
        while (true) {
            int event = parser.next();
            if (event == XMLStreamConstants.END_DOCUMENT) {
                parser.close();
                break;
            }

            if (event == XMLStreamConstants.START_ELEMENT) {
                String tag = parser.getLocalName();

                if (tag.equals("Cluster")) {
                    isCluster = true;
                    clsID = parser.getAttributeValue(0);
                    semType = parser.getAttributeValue(1);
                } else if (tag.equals("Entry") && isCluster) {
                    entryID = parser.getAttributeValue(0);
                    baseForm = parser.getAttributeValue(1);
                } else if (tag.equals("Variant") && isCluster) {

                    varWritten = parser.getAttributeValue(0);
                    varType = parser.getAttributeValue(1);

                    variants.add(varWritten + "~" + varType);
                } else if (tag.equals("DC") && isCluster) {
                    dcAtt = parser.getAttributeValue(0);
                    dcVal = parser.getAttributeValue(1);

                    dc.add(dcAtt + "~" + dcVal);
                }
            }

            if (event == XMLStreamConstants.END_ELEMENT && isCluster) {
                if (parser.getLocalName().equals("Cluster")) {
                    isCluster = false;
                    //clsid, entryid, semType, baseForm, variant(writtenform), variant(type), dc(att), dc(val)
                    // Use tabs as delimiter for Postgre COPY
                    String outStr = clsID + "/t" + entryID + "/t" + semType + "/t" + baseForm + "/t";

                    /* Add all variants */
                    for (String var : variants) {
                        String tmp[] = var.split("~");
                        varWritten = tmp[0];
                        varType = tmp[1];
                        outStr += varWritten + "/t" + varType + "/t";
                    }
                    /* Add al DCs */
                    for (String ss : dc) {
                        String[] tmp = ss.split("~");
                        dcAtt = tmp[0];
                        dcVal = tmp[1];
                        outStr += dcAtt + "/t" + dcVal + "/t";
                    }
                    // remove last tab "\t"
                    outStr = outStr.substring(0, outStr.length() - 2);
                    out.write(outStr);
                    variants.clear();
                    dc.clear();

                }
            }
        }

        // close all streams
        fstream.close();
        out.close();
        outStream.close();
    }
}

我格式化了你输入的。所以输入文件看起来像这样：

<Cluster clsId="UNIPR_NIRI_PARDP" semType="geneProt">
    <Entry entryId="UNIPR_NIRI_PARDP_1" baseForm="Protein nirI" type="PREFERRED">
        <Variant WRITTENFORM="FMN-binding domain protein" type="orthographic"/>
        <Variant WRITTENFORM="FMN-binding domain-containing protein" type="orthographic"/>
        <Variant WRITTENFORM="unknown" type="orthographic"/>
        <Variant WRITTENFORM="FMN-binding" type="orthographic"/>
        <Variant WRITTENFORM="Pden_2486" type="orthographic"/>
        <Variant WRITTENFORM="nirI" type="orthographic"/>
        <SourceDC sourceName="BioThesaurus" sourceId="Q51699"/>
        <PosDC posName="POS" pos="N"/>
        <DC att="uniprot_ac" val="Q51699"/>
        <DC att="speciesNameNCBI" val="318586"/>
    </Entry>
</Cluster>

输出看起来像这个。注意，它是用制表符分隔的。制表符稍后将用作Postgre COPY命令中的分隔符。您可以将分隔符更改为任何其他分隔符。

UNIPR_NIRI_PARDP/tUNIPR_NIRI_PARDP_1/tgeneProt/tProtein nirI/tFMN-binding domain protein/torthographic/tFMN-binding domain-containing protein/torthographic/tunknown/torthographic/tFMN-binding/torthographic/tPden_2486/torthographic/tnirI/torthographic/tuniprot_ac/tQ51699/tspeciesNameNCBI/t318586