RFC 4180対応版 CSVレコードの分解
Posted feedbacks - Flatten
Nested Hidden1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 | using System;
using System.Collections.Generic;
using System.Text;
class Program
{
static void Main()
{
string s = "\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx";
string[] ss = splitCSV(s);
for (int i = 0; i < ss.Length; ++i)
Console.WriteLine("{0} => {1}",i+1,ss[i]);
}
static string[] splitCSV(string s)
{
if (s == null) return null;
List<string> a = new List<string>();
int i = 0, j;
string t;
StringBuilder h = new StringBuilder();
while (i < s.Length)
{
bool b = s[i] == '"';
if (b) ++i;
j = s.IndexOf(b ? '"' : ',', i);
if (j < 0) j = s.Length;
t = s.Substring(i, j - i);
if (b && j < s.Length - 1 && s[j + 1] == '"')
{
h.Append(t);
h.Append('"');
i = j + 1;
}
else
{
a.Add(h + t);
h.Length = 0;
i = j + (b ? 2 : 1);
}
}
return a.ToArray();
}
}
|
Text::CSV_XSでさくっと．
1 | perl -MText::CSV_XS -le '$csv = Text::CSV_XS->new({binary=>1}); $csv->parse("\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx"); map { print ++$i, " => $_" } $csv->fields()'
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | program splitter;
{$APPTYPE CONSOLE}
uses
Classes;
procedure splitCSV(const Text: String);
var
i: Integer;
sl: TStringList;
begin
sl := TStringList.Create;
try
sl.CommaText := Text;
for i := 0 to sl.Count-1 do
Writeln(i+1, ' => ', sl[i]);
finally
sl.Free;
end;
end;
begin
splitCSV('"aaa","b'#13#10'bb","ccc",zzz,"y""Y""y",xxx');
end.
|
1 2 3 4 5 6 7 8 | require 'csv'
def splitCSV(str)
CSV::Reader.parse(str) do |x|
i=0; x.each{|d| puts "#{i.succ} => #{d}" }
end
end
splitCSV('"aaa","b
bb","ccc",zzz,"y""Y""y",xxx')
|
これでいいのかな？
1 2 3 4 5 6 7 8 9 | import csv
from StringIO import StringIO
data = """"aaa","b
bb","ccc",zzz,"y""Y""y",xxx"""
for row in csv.reader(StringIO(data)):
for i,v in enumerate(row):
print i+1, "=>", v
|
rubikitch
#1514()
[
Common Lisp
]
arnesi:parse-csv-stringはyYyとなるバグがあって使えない
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | (require :fare-csv)
(defun splitCSV (line)
(loop for elt in (with-input-from-string (inn line)
(fare-csv:read-csv-line inn))
for i from 1 do
(format t "~a => ~a~%" i elt)))
(splitCSV "\"aaa\",\"b
bb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx
")
;; 1 => aaa
;; 2 => b
;; bb
;; 3 => ccc
;; 4 => zzz
;; 5 => y"Y"y
;; 6 => xxx
|
1 2 3 4 5 6 7 8 9 10 11 12 13 | (use text.csv)
(use srfi-42)
(define (parse-csv line)
(call-with-input-string line
(lambda (port) ((make-csv-reader #\,) port))))
(define (splitCSV line)
(do-ec (: elt (parse-csv line))
(: i 1)
(format #t "~a => ~a~%" i elt)))
(splitCSV "\"aaa\",\"b
bb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx
")
|
kkobayashi
#1516()
[
R
]
Rはこういうの得意で、外部ライブラリーは不要です。 read.csvだとうまく行かなかったのでread.tableを使っています。
1 2 3 4 5 6 7 | splitCSV <- function(str){
table <- read.table(textConnection(str), sep=",", colClasses="character")
cat(paste(1:length(table), rep("=>", length(table)), table), sep="\n")
}
splitCSV('"aaa","b
bb","ccc",zzz,"y""Y""y",xxx')
|
1 2 3 4 5 6 7 8 9 10 11 12 13 | (use text.csv)
(define (splitCSV str)
(define (numberling l)
(let loop ((l l) (n 1))
(unless (null? l)
(format #t "~d => ~a~%" n (car l))
(loop (cdr l) (+ n 1)))))
(call-with-input-string str
(lambda (in)
(port-for-each
(pa$ numberling)
(pa$ (make-csv-reader #\,) in)))))
|
ｃｓｖモジュールを使うのが常道だと思いますが、あえて自前で処理してみました。cStringIOは初めて使いましたが、undoみたいな処理をするには便利かも。（seek の前の if c: が必要なのに気づかず少しはまりましたが）
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | import cStringIO
import os
def split_csv(csv):
io = cStringIO.StringIO(csv)
while not io.closed:
def f():
quoted = False
while 1:
c = io.read(1)
if not c: # eof
io.close()
break
elif not(quoted) and c == ',':
break
elif not(quoted) and c in ('\r', '\n'):
io.close() # ignore second record
break
elif c == '"':
c = io.read(1)
if c == '"':
yield c
else:
quoted = not(quoted)
if c:
io.seek(-1, os.SEEK_CUR)
else:
yield c
if quoted:
raise ValueError("unterminated quotation")
yield "".join(f())
def main():
for i, s in enumerate(split_csv("""\
"aaa","b
bb","ccc",zzz,"y""Y""y",xxx
""")):
print "%d => %s" % (i + 1, s)
if __name__ == '__main__':
main()
|
call-with-input-stringには「portを引数に取る手続き」であれば何でも渡せるので、 (make-csv-reader #\,) の結果をそのまま渡すことができます。 lambdaにくるむ必要はありません。 それから、単純なインデックスつきループであれば for-each-with-indexというのがあります。 なので、こんなふうに書けます：
1 2 3 4 5 6 | (use text.csv)
(use gauche.sequence)
(define (splitCSV line)
(for-each-with-index (cut print <>" => "<>)
(call-with-input-string line (make-csv-reader #\,))))
|
おっと、カラムの表示は1からスタートでしたか。そしたらcutは使えないですね。
1 2 3 4 5 6 7 | (use text.csv)
(use gauche.sequence)
(define (splitCSV line)
(for-each-with-index
(lambda (i e) (print (+ i 1)" => "e))
(call-with-input-string line (make-csv-reader #\,))))
|
【これはひどい】
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | (defun splitCSV (csv)
(let ((len (length csv))
(lst '(())))
(labels
((field (h n i &optional (esc nil))
(when (> len i)
(if h
(princ (format nil "~D => " n)))
(let ((it (char csv i)))
(case it
(#\" (if esc
(case (char csv (incf i))
(#\" (princ #\" ) (field nil n (1+ i) t))
(#\, (princ #\newline) (field t (1+ n) (1+ i))))
(field nil n (1+ i) t)))
(#\newline (if esc
(progn (princ it) (field nil n (1+ i) t))
(progn (princ it) (field t 1 (1+ i)))))
(#\, (princ #\newline) (field t (1+ n) (1+ i)))
(t (princ it) (field nil n (1+ i) esc)))))))
(field t 1 0 nil))))
|
まずは愚直にparseしてみた。
mapM putStrLn のあたりが気持ち悪い?
showField中で出力しちゃう方がいいんだろうか。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | import List
parseF [] r = reverse r
parseF ('\"':cs) r = parseE cs [] r
parseF cs r = parseN cs [] r
parseE [] f r = parseF [] $ reverse f:r
parseE ('\"':',':cs) f r = parseF cs $ reverse f:r
parseE ('\"':'\n':cs) f r = parseF cs $ reverse f:r
parseE ('\"':'\"':cs) f r = parseE cs ('\"':f) r
parseE ('\"':c:cs) f r = parseE ('\"':cs) f r -- should be an error?
parseE (c:cs) f r = parseE cs (c:f) r
parseN [] f r = parseF [] $ reverse f:r
parseN ('\n':cs) f r = parseF [] $ reverse f:r
parseN (',':cs) f r = parseF cs $ reverse f:r
parseN (c:cs) f r = parseN cs (c:f) r
splitCVS record = mapM putStrLn $ snd $ mapAccumL showField 1 $ parseF record []
where showField i f = (i+1, (show i)++" => "++f)
main = do splitCVS "\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx\n"; return ()
|
寝る前にもいっちょ。今度は自分でパース。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | import scala.collection.mutable.ListBuffer
def parseCSV(s:String):Array[Array[String]] = {
def split(s:String, c:String) = {
val buf = new ListBuffer[String]
val result = new ListBuffer[String]
s.split(c).foreach(p => p.filter(_ =='"').length%2 match{
case 0 if buf.isEmpty => result += p
case 0 if !buf.isEmpty => buf += p
case 1 if buf.isEmpty => buf += p
case 1 if !buf.isEmpty =>
buf += p
result += buf.mkString(c)
buf.clear
})
result
}
split(s,"\n").map(line => {
split(line, ",").map(col => {
col.replaceAll("\"\"", "\"").replaceAll("^(\")", "")
.replaceAll("(\")$", "")
}).toArray
}).toArray
}
val data = """"aaa","b
bb","ccc",zzz,"y""Y""y",xxx"""
parseCSV(data).foreach(line => {
(1 to line.length).foreach(i => {
println(i + " => " + line(i-1))
})
})
|
つづいてParsecに挑戦
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 | import Text.ParserCombinators.Parsec
record = do fields <- sepBy field (char ',')
char '\n'
return fields
field = between (char '\"') (char '\"') quotedField
<|> many (noneOf ",\n")
quotedField = many $ (try $ do string "\"\""; return '\"') <|> noneOf "\""
splitCVS line = case (runParser record () "" line) of
Left err -> print err
Right fields -> do mapM putStrLn fields; return ()
main = splitCVS "\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx\n"
|
お題に対応する最小限のチェックをしたつもりですが 入力文字列によっては見逃しがあるかも。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | def splitCVS(s):
a = []
b = s.split(',')
while b:
c = b.pop(0)
while c.startswith('"') and c.count('"') % 2:
c += b.pop(0)
if c.startswith('"') and c.endswith('"'):
c = c[1:-1].replace('""', '"')
elif c.find('"') != -1 or c.find('\n') != -1:
raise 'invalid'
a.append(c)
return a
l = splitCVS('"abc","b\nbb","cc,c",zzz,"y""Y""y",xxx')
for i, s in zip(range(len(l)), l):
print '%2d = %s' % (i+1, s)
|
一旦','で分割してしまってから'"'が奇数個含まれるフィールドに偶数になるまで後ろのフィールドを繋ぎ直す様な感じでやってみた。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | <?php
function splitcsv($line){
$r=array();
$a=explode(",",$line);
while(list(,$v)=each($a))
{ if(strpos($v,'"')!==false)
{ while(substr_count($v,'"')&1)
{ if(!(list(,$v1)=each($a)))
return false;
$v.=','.$v1;
}
ereg('"(.*)"',$v,$regs);
$v=str_replace('""','"',$regs[1]);
}
$r[]=$v;
}
return $r;
}
$line='"aaa","b
bb","ccc",zzz,"y""Y""y",xxx';
print_r(splitcsv($line));
?>
|
Parsec の出番！UnitTest 付きで。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | import Text.ParserCombinators.Parsec
import Test.HUnit
line :: Parser [String]
line = do columns <- sepBy1 column comma
optional (char '\n')
return columns
column :: Parser String
column = do b <- char '"'
c <- many ((satisfy (/= '"')) <|> try escapedQuote)
d <- char '"'
return c
<|>
many1 (noneOf [',', '"', '\n'])
<?> "escapedQuote"
escapedQuote :: Parser Char
escapedQuote = do string "\"\"" <?> "escapedQuote"
return '"'
comma :: Parser ()
comma = skipMany1 (char ',' <?> "comma")
splitCSV :: String -> [String]
splitCSV s = case (parse line "" s) of
Left err -> error ("parse error at " ++ (show err))
Right x -> x
testData :: [Test]
testData = [
["abc", "def"] ~=? splitCSV "abc,def\n",
["abc", "def"] ~=? splitCSV "\"abc\",def",
["a,bc", "def"] ~=? splitCSV "\"a,bc\",\"def\"\n",
["abc", "b\nbb", "ccc", "zzz", "y\"Y\"y", "xxx"] ~=? splitCSV "\"abc\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx\n"
]
main :: IO Counts
main = runTestTT (test testData)
|
あ、カラム番号とともに出力するところまでが課題なのね。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | import Text.ParserCombinators.Parsec
import Test.HUnit
line :: Parser [String]
line = do columns <- sepBy1 column comma
optional (char '\n')
return columns
column :: Parser String
column = do b <- char '"'
c <- many ((satisfy (/= '"')) <|> try escapedQuote)
d <- char '"'
return c
<|>
many1 (noneOf [',', '"', '\n'])
<?> "escapedQuote"
escapedQuote :: Parser Char
escapedQuote = do string "\"\"" <?> "escapedQuote"
return '"'
comma :: Parser ()
comma = skipMany1 (char ',' <?> "comma")
splitCSV :: String -> [String]
splitCSV s = case (parse line "" s) of
Left err -> error ("parse error at " ++ (show err))
Right x -> x
testData :: [Test]
testData = [
["abc", "def"] ~=? splitCSV "abc,def\n",
["abc", "def"] ~=? splitCSV "\"abc\",def",
["a,bc", "def"] ~=? splitCSV "\"a,bc\",\"def\"\n",
["abc", "b\nbb", "ccc", "zzz", "y\"Y\"y", "xxx"] ~=? splitCSV "\"abc\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx\n"
]
main :: IO ()
main = do cs <- getContents
mapM_ output (zip [1..] $ splitCSV cs)
where output (n, col) = putStrLn $ (show n) ++ " => " ++ col
|
無理矢理、正規表現でやってみました。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | import java.util.regex.*;
import java.util.*;
public class Sample {
private static final Pattern spliter =
Pattern.compile("((\"[^\"]*+\")+|[^,]*+),?");
private static final Pattern doubleQuote = Pattern.compile("\"\"");
public static String[] splitCSV(String rec) {
Matcher m = spliter.matcher(rec);
ArrayList<String> cols = new ArrayList<String>();
while (m.find()) {
String col = m.group(1);
if (col != null) {
if (col.startsWith("\"")) {
col = col.substring(1, col.length() - 1);
}
col = doubleQuote.matcher(col).replaceAll("\"");
cols.add(col);
}
if (m.end() >= rec.length())
break;
}
return cols.toArray(new String[cols.size()]);
}
public static void main(String[] args) throws Exception {
String sample = "\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx";
String[] cols = splitCSV(sample);
for (int i = 0; i < cols.length; i++) {
System.out.printf("%d => %s%n", i + 1, cols[i]);
}
}
}
|
Squeak Smalltalk で。 手近に CSV 解析器が見あたらなかったので、よく似た作業をする Smalltalk 処理系の字句解析器のインスタンスをハックして なんちゃって CSV 解析器(^_^;)を仕立ててみました。 具体的には、カンマをデリミタに、スペースを通常の文字に、 CR を閉じ括弧、LF を開く括弧に見立てるよう、スキャナのテーブルを 書き換え騙して仕事をさせます。なお、ダブルクオートは Smalltalk では コメントアウトになってしまうので、文字列リテラルを表す シングルクオートに差し替え、解析後、ダブルクオートに戻しています。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | | scanner typeTable data dataFile dataString |
scanner := Scanner new.
typeTable := scanner instVarNamed: #typeTable.
typeTable := scanner instVarNamed: #typeTable put: typeTable copy.
typeTable at: $, asciiValue put: #xDelimiter.
typeTable at: $ asciiValue put: #xLetter.
typeTable at: Character lf asciiValue put: #leftParenthesis.
typeTable at: Character cr asciiValue put: #rightParenthesis.
dataFile := FileStream fileNamed: 'data.txt'.
dataString := dataFile contents replaceAll: $" with: $'; copyWithFirst: $(.
data := scanner scanTokens: dataString.
World findATranscript: nil.
data do: [:record |
record doWithIndex: [:field :index |
field replaceAll: $' with: $".
field := field copyWithout: Character lf.
Transcript cr; show: ('{1} => {2}' format: {index. field})]]
|
csvをつかっても面白くないので正規表現でやってみました。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | import re
quoted = r'("((""|[^"])+)")'
naked = r'([^,"\n]+)'
enclosed = r'("(?P<enclosed>' + naked + r')")'
record = quoted + '|' + enclosed + '|' + naked
r = re.compile(record)
def unescape(s):
return re.sub('["](?!")', '', s)
def parse(s):
for i, t in enumerate(r.finditer(s)):
print i+1, '=>', unescape(s[t.start():t.end()])
parse('''"aaa","b
bb","ccc",zzz,"y""Y""y",xxx''')
|
なんか微妙。 $ ./a.out < data のような感じで実行するのを想定しています。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | #include <stdio.h>
#include <string.h>
enum { false, true };
int print_one_value();
int main()
{
int c;
int ret;
int count = 1;
while (true) {
printf("%d => ", count);
ret = print_one_value();
puts("");
if (ret) {
break;
}
count++;
}
return 0;
}
int print_one_value(int count)
{
int c;
int quote_in = false;
int quote_before = false;
c = getchar();
if (c == '"') {
quote_in = true;
}
else {
putchar(c);
}
while ((c = getchar()) != EOF) {
if (c == ',') return false;
else if (c == '\n' && !quote_in) return true;
if ((c == ',' || c == '\n' || c == '"')
&& quote_before) {
putchar(c);
quote_before = false;
}
else if (c == '"') {
quote_before = true;
}
else {
putchar(c);
quote_before = false;
}
}
}
|
""""の処理がおかしかった。
unescapeが美しくない。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | '''
>>> parse('aaa')
1 => aaa
>>> parse('"aaa"')
1 => aaa
>>> parse('"a\\naa"')
1 => a\naa
>>> parse('"a""aa"')
1 => a"aa
>>> parse('"a""""aa"')
1 => a""aa
>>> parse('aaa,bbb')
1 => aaa
2 => bbb
>>> parse('aaa, bbb')
1 => aaa
2 => bbb
>>> parse('aaa,"b\\nbb"')
1 => aaa
2 => b\nbb
>>> parse('aaa,"b\\n""bb"')
1 => aaa
2 => b\n"bb
'''
import re
quoted = r'("((""|[^"])+)")'
naked = r'([^,"\n]+)'
enclosed = r'("(?P<enclosed>' + naked + r')")'
record = quoted + '|' + enclosed + '|' + naked
r = re.compile(record)
def unescape(s):
if s.startswith('"'):
return re.sub('""', r'"', s[1:-1])
else:
return re.sub('""', r'"', s)
def parse(s):
for i, t in enumerate(r.finditer(s)):
print i+1, '=>', unescape(s[t.start():t.end()])
parse('''"aaa","b
b""b","ccc",zzz,"y""Y""y",xxx''')
import doctest
doctest.testmod()
|
すでに何人か指摘してくれていますが、
サンプルのデータが足りなかったですねorz
テストするときには「,"eee,EEE",,,」
というのも最後に付け足してあげてください。
名前つきの正規表現を使ってみた。
get orの連続とrecordの定義が重複していて美しくない。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | '''
>>> parse('aaa')
1 => aaa
>>> parse('"aaa"')
1 => aaa
>>> parse('"a\\naa"')
1 => a\naa
>>> parse('"a""aa"')
1 => a"aa
>>> parse('"a""""aa"')
1 => a""aa
>>> parse('aaa,bbb')
1 => aaa
2 => bbb
>>> parse('aaa, bbb')
1 => aaa
2 => bbb
>>> parse('aaa,"b\\nbb"')
1 => aaa
2 => b\nbb
>>> parse('aaa,"b\\n""bb"')
1 => aaa
2 => b\n"bb
'''
import re
quoted = r'("(?P<quoted>(""|[^"])+)")'
naked = r'[^,"\n]+'
enclosed = r'("(' + '?P<enclosed>' + naked + '' + r')")'
record = quoted + '|' + enclosed + '|' + '(?P<naked>' + naked + ')'
r = re.compile(record)
def unescape(s):
return re.sub('""', r'"', s)
def parse(s):
for i, t in enumerate(r.finditer(s)):
#print i+1, '=>', unescape(s[t["body"].start:t["body"].end])
d= t.groupdict()
print i+1, '=>', unescape(d.get('naked') or d.get('enclosed') or d.get('quoted'))
parse('''"aaa","b
b""b","ccc",zzz,"y""Y""y",xxx''')
import doctest
doctest.testmod()
|
kkobayashi
#1553()
[
R
]
念のため、実行結果です（>と+はR Consoleのプロンプトです）
1 2 3 4 5 6 7 8 9 10 11 12 13 | > splitCSV('"aaa","b
+ bb","ccc",zzz,"y""Y""y",xxx,"eee,EEE",,,')
1 => aaa
2 => b
bb
3 => ccc
4 => zzz
5 => y"Y"y
6 => xxx
7 => eee,EEE
8 =>
9 =>
10 =>
|
色々修正版 (splitCSV "\"aaa\",\"b bb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx,\"eee,EEE\",,,") 1 => aaa 2 => b bb 3 => ccc 4 => zzz 5 => y"Y"y 6 => xxx 7 => eee,EEE 8 => 9 => nil
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | (defun splitCSV (csv)
(let ((len (length csv)))
(labels
((field (h n i &optional (esc nil))
(when (> len i)
(and h (princ (format nil "~D => " n)))
(let ((it (char csv i)))
(case it
(#\" (if esc
(case (char csv (incf i))
(#\" (princ #\" ) (field nil n (1+ i) t))
(#\, (princ #\newline) (field t (1+ n) (1+ i))))
(field nil n (1+ i) t)))
(#\newline (if esc
(progn (princ it) (field nil n (1+ i) t))
(progn (princ it) (field t 1 (1+ i)))))
(#\, (if esc
(progn (princ it) (field nil n (1+ i) t))
(progn (princ #\newline) (field t (1+ n) (1+ i)))))
(t (princ it) (field nil n (1+ i) esc)))))))
(field t 1 0 nil))))
|
やっちゃった＞＜ 相互呼び出しの再帰で実装してみた。 これって、10 => まで出るのが正しいんだよね？ [sample.csv] "aaa","b bb","ccc",zzz,"y""Y""y",xxx,"eee,EEE",,, (splitCSV "sample.csv") 1 => aaa 2 => b bb 3 => ccc 4 => zzz 5 => y"Y"y 6 => xxx 7 => eee,EEE 8 => 9 => 10 => t
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | (defun splitCSV (csv-file)
(labels
((in-esc (csv header col)
(let ((it (read-char csv nil)))
(case it
((#\") (let ((it (read-char csv nil)))
(case it
(#\" (princ it) (in-esc csv nil col))
(#\, (princ #\newline) (out-esc csv t (1+ col))))))
((nil) nil)
(t (princ it) (in-esc csv nil col)))))
(out-esc (csv header col)
(let ((it (read-char csv nil)))
(and header it (format t "~D => " col))
(case it
((#\") (in-esc csv nil col))
((#\newline) (princ it) (out-esc csv t 1))
((#\,) (princ #\newline) (out-esc csv t (1+ col)))
((nil) t)
(t (princ it) (out-esc csv nil col))))))
(with-open-file (csv csv-file)
(out-esc csv t 1))))
|
匿名
#1581()
[
JavaScript
]
JavaScriptでの投稿が無いようなので。ちょっと修正すれば複数レコードのパーズも対応できるはず。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | function unquote(val) {
return (val.indexOf('"') == 0) ? val.substring(1, val.length-1).replace(/""/g, '"')
: val;
}
function splitCsv(csv) {
// カラム分離用パターン
// グループ1⇒ 行頭 or カンマ or 改行
// グループ2⇒ 空文字列 or 非quoted文字列 or quoted文字列
// 非quoted文字列⇒ 改行, カンマ, " を含まない1文字 +
// 改行, カンマを含まない文字を0文字以上
// quoted文字列⇒ " + ( "" or " 以外の1文字 ) を0回以上 + "
// (?=,|\\r?\\n|$)⇒ 次に カンマ, 改行, 行末が続くこと(幅0肯定先読み)
var reg = new RegExp('(^|,|\\r?\\n)(|[^"\\r\\n,][^\\r\\n,]*|(?:"(?:""|[^"])*"))(?=,|\\r?\\n|$)', 'g');
var ary = [];
var match = null;
while(match = reg.exec(csv)) {
ary.push(unquote(match[2]));
}
return ary;
}
function printCsv(input) {
var values = splitCsv(input);
var ary = [];
for(var i=0; i<values.length; i++) {
ary.push((i+1) + ' => ' + values[i]);
}
alert(ary.join("\n"));
}
printCsv('"aaa","b\n\
bb","ccc",zzz,"y""Y""y",xxx');
|
Scala 2.6.0-RC1でscala.util.parsing.combinatorパッケージが標準ライブラリになりました。 ということでScalaでパーサコンビネータ。ほとんど資料がないので手探りですが。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | import scala.util.parsing.combinator.{Parsers, ImplicitConversions, ~, mkTilde}
import scala.util.parsing.input.CharArrayReader
import Character.isISOControl
object CSVParser {
trait Base
case class Field(s:String) extends Base {
override def toString = s
}
case class Record(fields: List[Field]) extends Base
case class File(records :List[Record]) extends Base
def mkString(cs :List[Any]) = cs.mkString("")
class CSVParser extends Parsers {
type Elem = Char
def notMeta(c:Elem) = c!=',' && c!='\n' && c!='"' && !isISOControl(c)
lazy val file = record.*('\n') ^^ File
lazy val record = (field|quotedField|nullableField).*(',') ^^ Record
lazy val field = chars.+ ^^ {cs => Field(mkString(cs))}
lazy val nullableField = chars.* ^^ {cs => Field("")}
lazy val quotedField = '"' ~ (charsInQuote|quoteInQuote).* ~ '"' ^^ {cs => Field(mkString(cs))}
lazy val charsInQuote = elem("chars in field", _!='"')
lazy val quoteInQuote = repN(2, quote) ^^ {cs => '"'}
lazy val quote = '"' ^^ success
lazy val chars = elem("chars", notMeta)
}
}
val data = """
"aaa","b
bb","ccc",zzz,"y""Y""y",xxx
""".trim
(new CSVParser.CSVParser).file(new
CharArrayReader(data.toCharArray)).map(file => {
file.records.map({record =>
val fields = record.fields
(1 to fields.length).foreach(i => println(i +" => " + fields(i-1)))
})
})
|
ファイルからの読み込みまでやってみました。
あと、CSVデータを列データに別ける部分がおかしかったので修正しています。("の中の'\n'と','の取り扱い方)
実行結果
% cat data.txt
"aaa","b
bb","ccc",zzz,"y""Y""y",xxx,"eee,EEE",,,
dddd,eee
% ./a.out data.txt
1 => aaa
2 => b
bb
3 => ccc
4 => zzz
5 => y"Y"y
6 => xxx
7 => eee,EEE
8 =>
9 =>
10 =>
1 => dddd
2 => eee
1 =>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | #include <stdio.h>
enum { false, true, eof };
int print_one_value(FILE *fp);
int main(int argc, char **argv)
{
int ret;
int count = 1;
FILE *fp;
if (argc < 2) {
fprintf(stderr, "usage: %s file_name\n", argv[0]);
return 2;
}
fp = fopen(argv[1], "r");
if (fp == NULL) {
fprintf(stderr, "Error\n");
return 1;
}
while (true) {
printf("%d => ", count);
ret = print_one_value(fp);
puts("");
if (ret == eof) {
break;
}
else if (ret == true) {
count = 0;
}
count++;
}
fclose(fp);
return 0;
}
int print_one_value(FILE *fp)
{
int c;
int quote_in = false;
while ((c = fgetc(fp)) != EOF) {
if (c == ',' && (!quote_in)) return false;
int quote_in = false;
while ((c = fgetc(fp)) != EOF) {
if (c == ',' && (!quote_in)) return false;
else if (c == '\n' && (!quote_in)) return true;
if (c == '"') {
c = fgetc(fp);
if (c == '"' && quote_in) {
putchar(c);
}
else {
quote_in ^= 1;
ungetc(c, fp);
}
}
else {
putchar(c);
}
}
return eof;
}
|
題意に沿うバージョンと、ファイルから読み込むバージョンとに分けてみた。 ＃終端処理が微妙なことに気づいたので微妙に修正…
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | (defun splitCSV (csv-stream)
(labels
((in-esc (csv header col)
(let ((it (read-char csv nil)))
(case it
((#\") (let ((it (read-char csv nil)))
(case it
(#\" (princ it) (in-esc csv nil col))
(#\, (princ #\newline) (out-esc csv t (1+ col))))))
((nil) nil)
(t (princ it) (in-esc csv nil col)))))
(out-esc (csv header col)
(let ((it (read-char csv nil)))
(and header (or (> col 1) it) (format t "~D => " col))
(case it
((#\") (in-esc csv nil col))
((#\newline) (princ it) (out-esc csv t 1))
((#\,) (princ #\newline) (out-esc csv t (1+ col)))
((nil) t)
(t (princ it) (out-esc csv nil col))))))
(and (streamp csv-stream)
(out-esc csv-stream t 1))))
(defun splitCSV-from-file (csv-file)
(with-open-file (stream csv-file)
(splitCSV stream)))
(defun splitCSV-from-string (csv-string)
(with-input-from-string (stream csv-string)
(splitCSV stream)))
|
[1..100]>>=pen
#1639()
[
Haskell
]
mapAccumL を使ってみました。 状態遷移表という感じ。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | import List
import Maybe
data State = F | E | G | Z
parse (F,xs) '\"' = ((E,xs), Nothing)
parse (F,xs) '\n' = ((Z,[]), Just xs)
parse (F,xs) ',' = ((F,[]), Just xs)
parse (F,xs) c = ((F,xs++[c]), Nothing)
parse (E,xs) '\"' = ((G,xs), Nothing)
parse (E,xs) c = ((E,xs++[c]), Nothing)
parse (G,xs) ',' = ((F,[]), Just xs)
parse (G,xs) '\n' = ((F,[]), Just xs)
parse (G,xs) '\"' = ((E,xs++"\""), Nothing)
parse (G,xs) c = ((F,xs++[c]), Nothing)
parse (Z,_) c = ((Z,[]), Nothing)
splitCVS record = mapM_ putStrLn $ zipWith showFiled [1..]
$ case s of
(F, xs) -> fs ++ [xs]
(E, xs) -> fs ++ [xs]
_ -> fs
where
showFiled i f = (show i)++" => "++f
(s, xss) = mapAccumL parse (F,[]) record
fs = catMaybes xss
|
あまり自信ありませんが、flexです。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | %option main
%x q
%{
int i = 1;
int start = 1;
void p(char c)
{
if (start) { start = 0; printf("%d => ", i++); }
putchar(c);
}
%}
%%
<INITIAL>\" { BEGIN(q); }
<INITIAL>, { p('\n'); start = 1; }
<INITIAL>"\n" { p('\n'); start = 1; i = 1; }
<q>\"\" { p('\"'); }
<q>\" { BEGIN(INITIAL); }
<*>.|\n { p(*yytext); }
%%
|
[1..100]>>=pen
#1683()
[
Haskell
]
入力文字列を Maybe列（Nothing が EOF的役割）に変換してから 処理させてコードをすっきりさせました。 さらに Maybe を組み合わせて '\n' を認識したところで処理を 打ち切るようにさせたいのですが Maybe の嵐になるのでこのへんで やめておきます。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 | import List
import Maybe
data State = F | E | G | Z
parse (F,xs) (Just '\"') = ((E,xs), Nothing)
parse (F,xs) (Just '\n') = ((Z,[]), Just xs)
parse (F,xs) (Just ',') = ((F,[]), Just xs)
parse (F,xs) (Just c) = ((F,xs++[c]), Nothing)
parse (F,xs) Nothing = ((Z,[]), Just xs)
parse (E,xs) (Just '\"') = ((G,xs), Nothing)
parse (E,xs) (Just c) = ((E,xs++[c]), Nothing)
parse (E,xs) Nothing = ((Z,[]), Just xs)
parse (G,xs) (Just ',') = ((F,[]), Just xs)
parse (G,xs) (Just '\n') = ((F,[]), Just xs)
parse (G,xs) (Just '\"') = ((E,xs++"\""), Nothing)
parse (G,xs) (Just c) = ((F,xs++[c]), Nothing)
parse (G,xs) Nothing = ((Z,[]), Just xs)
parse (Z,_) _ = ((Z,[]), Nothing)
splitCVS record = mapM_ putStrLn $ zipWith showFiled [1..]
$ catMaybes $ snd
$ mapAccumL parse (F,[]) $ map Just record ++ [Nothing]
where
showFiled i f = (show i)++" => "++f
|
手抜き版。相互再帰lexer
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | main = putStr
. unlines
. map (uncurry $ flip ((.) . (++) . show) (" => "++))
. zip [1..]
. splitCSV =<< getContents
splitCSV = lex0 [] ""
lex0 cs c "" = reverse (c:cs)
lex0 cs c ('"' :xs) = lex1 cs c xs
lex0 cs c (',' :xs) = lex0 (reverse c:cs) "" xs
lex0 cs c (x :xs) = lex0 cs (x:c) xs
lex1 cs c ('"':',':xs) = lex0 (reverse c:cs) "" xs
lex1 cs c ('"':'"':xs) = lex1 cs ('"':c) xs
lex1 cs c (x :xs) = lex1 cs (x :c) xs
|
上のコードをC++に移植しました。カバレッジ稼ぎ。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | #include <iostream>
#include <vector>
#include <string>
#include <iterator>
std::vector<std::string> split_csv(const std::string& csv)
{
std::vector<std::string> v;
std::string::const_iterator it = csv.begin();
while (it != csv.end())
{
std::string s;
bool quoted = false;
while (it != csv.end())
{
if (!quoted && *it == ',')
{
++it;
break;
}
else if (!quoted && (*it == '\r' || *it == '\n'))
{
it = csv.end(); // ignore second record
break;
}
else if (*it == '"' && (++it == csv.end() || *it != '"'))
{
quoted = !quoted;
}
else
{
s.append(1, *it++);
}
}
v.push_back(s);
}
return v;
}
int main()
{
const char csv[] = "\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx";
std::vector<std::string> v = split_csv(csv);
for (size_t i = 0; i < v.size(); ++i)
{
std::cout << (i + 1) << " => " << v[i] << std::endl;
}
}
|
別途csvモジュールが必要。 http://pnuts.org/extensions/csv/
1 2 3 4 5 | use("csv")
for (columns: readCSV("csv.txt")){
i=0
for(c:columns) println(++i, " => ", c)
}
|
% cat test.csv "aaa","b bb","ccc",zzz,"y""Y""y",xxx "aaa","""","ccc",zzz,"y""Y""y",xxx,"u,v" % awk -f csv.awk test.csv 1 => aaa 2 => b bb 3 => ccc 4 => zzz 5 => y"Y"y 6 => xxx 1 => aaa 2 => " 3 => ccc 4 => zzz 5 => y"Y"y 6 => xxx 7 => u,v
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | BEGIN {
s = "" # バッファ
r = 0 # フィールド番号
t = 2 # バッファのどこから " を探すか（覚えておくため）
}
{
s = s $0
while (s !‾ /^$/) {
if (s ‾ /^"/) { # "で始まる
if (match(substr(s,t), /"/)) {
if (substr(s,t+RSTART-1) ‾ /^""/) { # 次が"ならスキップしたい
t += RSTART + 1 ; continue
}
# 閉じる"を検出.
} else {
# 閉じる"がなければ１行追加
getline nextline
s = s "¥n" nextline
continue
}
# "で囲まれた部分 ("" が含まれている場合がある)
s0 = substr(s,2,t+RSTART-3)
gsub(/""/,"¥"", s0)
printf("%d => %s¥n", ++r, s0)
s = substr(s, t+RSTART)
t = 2 # 戻しておく
if (s ‾ /^,/) {
s = substr(s,2)
} else if (s ‾ /^$/) {
printf("¥n")
r = 0
s = ""
}
} else {
if (match(s, /,/)) {
printf("%d => %s¥n", ++r, substr(s,1,RSTART-1))
s = substr(s,RSTART+1)
} else {
printf("%d => %s¥n", ++r, s)
s = ""
}
}
}
r = 0
printf "¥n"
}
|
PrologでDCG使って書いてみました。
?- print_record('"aaa","b\nbb","ccc",zzz,"y""Y""y",xxx').
1 => aaa
2 => b
bb
3 => ccc
4 => zzz
5 => y"Y"y
6 => xxx
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | csv([R|Rs]) --> record(R), "\n", csv(Rs), !.
csv([R]) --> record(R), !.
csv([]) --> [].
record([D|Ds]) --> field(D), ",", record(Ds), !.
record([D]) --> field(D), !.
record([]) --> [].
field(D) --> "\"", quoted(D), "\"" ; naked(D).
naked([C|Cs]) --> [C], { \+ member(C, "\",\n") }, naked(Cs), !.
naked([]) --> [].
quoted([0'"|Cs]) --> "\"\"", quoted(Cs), !.
quoted([C|Cs]) --> [C], { C \== 0'" }, quoted(Cs), !.
quoted([]) --> [].
print_record(Atom) :-
name(Atom, CSV),
phrase(csv([Record]), CSV),
forall(nth1(N, Record, Data), writef('%w => %s\n', [N, Data])).
|
汚いなぁ。。。 fiber 便利。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | Iterator::with_peek: method fiber {
prev, curr: null, null;
noloop: true;
this {|next|
if (curr) {
yield prev, curr, next;
}
prev, curr = curr, next;
noloop = false;
}
if (!noloop) {
yield prev, curr, null;
}
}
CSVParser: class {
- _iter;
- _finish;
initialize: method(string) {
_iter = string.split("").each.with_peek;
_finish = false;
}
flush: method(field) {
r: field.join("");
field.clear();
return r;
}
line_parser: method fiber {
in_quote: false;
field: [];
_iter {|prev,it,next|
if (in_quote) {
if (it == "\"") {
if (prev == "\"") {
// ignore
} else if (next == "\"") {
field.push_back(it);
} else {
in_quote = false;
}
} else {
field.push_back(it);
}
} else {
if (field.empty() && it == "\"") {
in_quote = true;
} else if (it == "\n") {
yield flush(field);
break;
} else if (it == ",") {
yield flush(field);
} else {
field.push_back(it);
}
}
} nobreak {
_finish = true;
}
if (!field.empty()) {
yield flush(field);
}
}
parse: method fiber {
while (!_finish) {
yield line_parser();
}
}
}
parser: CSVParser(
[%!"aaa","b\nbb","ccc",zzz,"y""Y""y",xxx!,
%!a,b,c,d!,
%!a,b,c,!,
%!a!,
%!!,
].join("\n"));
format: %f[%(line)d:%(col)d: %(cell)s];
parser.parse.with_index {|lineno,line|
line.with_index {|colno,it|
format(line: lineno, col: colno, cell: it).p;
}
}
|
あえて自力でやってみました。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | def parse_csv(d)
records = []
columns = []
buffer = ""
in_quote = false
i = 0
while i < d.size
if d[i] == ?"
if d[i+1] != ?"
in_quote = !in_quote
i += 1
next
else
i += 1
end
end
unless in_quote
if d[i] == ?, || d[i] == ?\n
columns << buffer
buffer = ""
if d[i] == ?\n
records << columns
columns = []
end
i += 1
next
end
end
buffer += d[i].chr
i += 1
end
records
end
data=<<EOT
"aaa","b
bb","ccc",zzz,"y""Y""y",xxx
EOT
records=parse_csv(data)
records.first.each_with_index{ |r,index|
puts "#{index+1} => #{r}"
}
|
昔書いたコードってホントよくわからないものですね(笑 サンプル出力するとうまく出るのでちゃんと機能しているはず^^；； "aaa","b bb","ccc","",zzz,"y""Y""y","xx,x",,, 1 => aaa 2 => b bb 3 => ccc 4 => 5 => zzz 6 => y"Y"y 7 => xx,x 8 => 9 => 10 =>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | sub splitCSV($@)
{
my $work = shift || ''; # CSV形式の１レコードの文字列
my $sepCHAR = shift || ','; # 区切り文字
my $quoteCHAR = shift || '"'; # 引用符
return 0 if( $work eq '' );
$work .= $sepCHAR; # レコードの最後に , を追加
# とりあえず列の取り出し;
my @result = ($work =~ m/((?:$quoteCHAR$quoteCHAR|$quoteCHAR.*?[^$quoteCHAR]$quoteCHAR)[ \t]*|(?:.*?))$sepCHAR/sg);
######################
# レコードのデータに
# 引用符 が含まれていた場合
# 引用符 を はずして 引用符二個で一つに変換する
if( $work =~ m/$quoteCHAR/ ){
my $count = 0;
foreach $work ( @result ){
#######################
# 取り出した列の整形
# ""のみはデータ無しで空白に置換
$work = '' if( !defined($work) || $work eq "$quoteCHAR$quoteCHAR" );
$work =~ s/$quoteCHAR(.+)$quoteCHAR/$1/s; # " で囲まれていた場合 "を取り外す
$work =~ s/$quoteCHAR$quoteCHAR/$quoteCHAR/g; # "" は " に変換
$result[ $count ++ ] = $work; # 変換後の結果で更新
}
}
return @result;
}
my @field;
my $record = <<CSV;
"aaa","b
bb","ccc","",zzz,"y""Y""y","xx,x",,,
CSV
print "$record\n";
@field = &splitCSV($record);
for( my $n=0; $n<=$#field; $n ++)
{
print "$n => $field[$n]\n";
}
|
自前パースの戦略です。
OCamlではこの手のパースはストリームパーサを利用すると少しだけ楽かもしれません。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | (* strの中にcharはあるか? *)
let mem_string char str =
try
let _ =
String.index str char
in
true
with
Not_found ->
false
(* 一文字バッファに貯めて次へ *)
let next buf strm loop =
match Stream.peek strm with
Some c ->
Buffer.add_char buf (Stream.next strm);
loop strm
| _ ->
if Buffer.length buf > 0 then
Buffer.contents buf
else
raise Stream.Failure
(* 区切り文字が出るまでバッファに文字を貯めていく *)
let rec until_sep ?(buf = Buffer.create 80) strm =
match Stream.peek strm with
Some c when mem_string c ",\r\n" ->
Buffer.contents buf
| _ ->
next buf strm (until_sep ~buf)
(* 括り文字が出るまでバッファに文字を貯めていく *)
let rec until_quote ?(buf = Buffer.create 80) strm =
match Stream.peek strm with
Some c when c = '"' -> begin
match Stream.npeek 2 strm with
'"' :: '"' :: [] ->
Buffer.add_char buf (Stream.next strm);
Stream.junk strm;
until_quote ~buf strm
| _ ->
Buffer.contents buf
end
| _ ->
next buf strm (until_quote ~buf)
(* 一つのフィールドを認識。括られている奴と括られていない奴 *)
let parse_field = parser
[< 'fq when fq = '"'; field = until_quote; 'sq when sq = '"' >] ->
field
| [< field = until_sep >] ->
field
(* フィールドを切り取りつつ表示 *)
let _ =
let print_field =
let counter =
ref 1
in
fun str ->
Printf.printf "%d => %s\n" !counter str;
incr counter
in
let rec parse = parser
[< field = parse_field; strm >] ->
print_field field;
begin match strm with parser
[< 'c when c = ','; rest >] ->
parse rest
| [< >] ->
()
end
| [< >] ->
()
in
parse (Stream.of_string "\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx\n")
|
Javaらしく（？）Readerクラスで処理してみました。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
public class Answer33 {
public static void main(String[] args) {
String str = "\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx";
CSVDataReader reader = new CSVDataReader(new StringReader(str));
try {
while (true) {
int cell = reader.getCellNumber();
String s = reader.readCell();
if (s == null) break;
System.out.println(cell + " => " + s);
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
class CSVDataReader extends BufferedReader {
private int cellCount_;
public CSVDataReader(Reader reader) {
super(reader);
cellCount_ = 1;
}
public int getCellNumber() {
return cellCount_;
}
public String readCell() throws IOException {
int c = read();
if (c < 0) return null;
cellCount_++;
StringBuilder builder = new StringBuilder();
boolean quote = (c == '"');
if (!quote) {
if (c == '\r' || c == '\n') return "";
builder.append((char) c);
}
OUTER: while ((c = read()) >= 0) {
if (quote) {
INNER: switch (c) {
case '"':
int next = read();
switch (next) {
case '"':
builder.append('"');
break INNER;
case ',':
break OUTER;
default:
throw new IllegalStateException();
}
default:
builder.append((char) c);
}
} else {
switch (c) {
case ',':
case '\r':
case '\n':
break OUTER;
case '"':
throw new IllegalStateException();
default:
builder.append((char) c);
}
}
}
return builder.toString();
}
}
|
Genlex という OCaml に付属の簡易字句解析モジュールを使ってみました。
非常に手抜きな作りなので、サンプルは正しく出力できますが、きちんと CSV に対応はしていません。
非常に手抜きな作りなので、サンプルは正しく出力できますが、きちんと CSV に対応はしていません。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | open Genlex
let string_of_token token =
match token with
| Ident s
| String s -> s
| Int i -> string_of_int i
| Float f -> string_of_float f
| Char c -> String.make 1 c
| _ -> failwith "not use Kwd"
let columns_of_tokens str =
let tokens = Genlex.make_lexer [","] (Stream.of_string str) in
let peek () = Stream.peek tokens
and junk () = Stream.junk tokens in
let rec loop acc =
match peek () with
| None -> List.rev acc
| Some (Kwd _) ->
junk ();
loop acc
| Some token ->
junk ();
let column = string_of_token token in
let rec concat col =
match peek () with
| None
| Some (Kwd _) -> col
| Some tok ->
junk ();
concat (col ^ "\"" ^ (string_of_token tok))
in
loop ((concat column) :: acc)
in
loop []
let parse_and_print str =
match columns_of_tokens str with
| [] -> print_newline ()
| x::xs ->
Printf.printf "1 => %s\n" x;
ignore begin
List.fold_left begin fun index str ->
Printf.printf "%d => %s\n" index str;
succ index
end 2 xs
end
let main () =
let sample =
match Sys.argv with
| [|_; input |] -> input
| _ -> "\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx"
in
parse_and_print sample
let () = if not !Sys.interactive then main ()
|
PEG (Parsing expression grammar) ベースのパターンマッチ OOPL である OMeta で。
OMeta には、COLA（Combined Object-Lambda Architecture; aka, Pepsi&Coke）、Squeak Smalltalk、JavaScript での実装がありますが、ここでは Squeak OMeta を用い、Doukaku33 として定義しました。
実行例
| in record |
in := '"aaa","b
bb","ccc",zzz,"y""Y""y",xxx'.
record := (Doukaku33 onTree: nil) apply: #レコード withArguments: in.
World findATranscript: nil.
record doWithIndex: [:field :idx | Transcript cr; show: idx; show: ' => ', field]
出力
1 => aaa
2 => b
bb
3 => ccc
4 => zzz
5 => y"Y"y
6 => xxx
OMeta には、COLA（Combined Object-Lambda Architecture; aka, Pepsi&Coke）、Squeak Smalltalk、JavaScript での実装がありますが、ここでは Squeak OMeta を用い、Doukaku33 として定義しました。
実行例
| in record |
in := '"aaa","b
bb","ccc",zzz,"y""Y""y",xxx'.
record := (Doukaku33 onTree: nil) apply: #レコード withArguments: in.
World findATranscript: nil.
record doWithIndex: [:field :idx | Transcript cr; show: idx; show: ' => ', field]
出力
1 => aaa
2 => b
bb
3 => ccc
4 => zzz
5 => y"Y"y
6 => xxx
see: OMeta: an Object-Oriented Language for Pattern Matching
1 2 3 4 5 | レコード ::= <列>:first ($, <列>)*:rest => [rest addFirst: first; yourself]
列 ::= (<クオートあり> | <クオートなし>):xs => [String withAll: xs]
クオートあり ::= $" ($" $" => [$"] | ~$" <char>)+:xs $" => [xs]
クオートなし ::= (~(<改行> | $, | $") <char>)*
改行 ::= <exactly (Character cr)>
|
単にデータを加工して出力しただけです. 題意を満たしているかどうか自信がありませんが.
1 2 3 4 5 6 7 8 9 10 11 | STR = <<EOS
"aaa","b
bb","ccc",zzz,"y""Y""y",xxx
EOS
def splitCSV(str)
str.split(/,/).
map{|x| x.match(/\A"?([^"](?:.|\s)+[^"])"?\z/)[1].gsub('""', '"')}.
each_with_index{|x, i| print "#{i+1} => #{x}\n"}
end
splitCSV(STR)
exit
|
PHP 入出力ストリームと fgetcsvを使って簡単にやってみた
1 2 3 4 5 6 7 8 9 10 11 12 13 | <?php
function csv2array($csv)
{
$fp = fopen('php://temp/maxmemory:'.(5*1024*1024), 'r+');
fputs($fp, $csv);
rewind($fp);
return fgetcsv($fp);
}
$csv='"aaa","b
bb","ccc",zzz,"y""Y""y",xxx';
var_dump(csv2array($csv));
|
CSV解析用ステートマシンつくってみました。 (とりあえずmain()では標準入力からとるようにしていますが、適時書き換えてください。) $ gcc main.c $ cat > csv.txt "aaa","b bb","ccc",zzz,"y""Y""y",xxx "aaa","b bb","cc c",zzz,"y""""""y",xxx $ ./a.out < csv.txt 1 => aaa 2 => b bb 3 => ccc 4 => zzz 5 => y"Y"y 6 => xxx 1 => aaa 2 => b bb 3 => cc c 4 => zzz 5 => y"""y 6 => xxx
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 | /*-
* The MIT License
*
* Copyright (c) 2008 虹原いんく
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
/* csv.state : http://ja.doukaku.org/33/ 寄稿用 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* \<TAB> \, を無視する。 = 1 */
#define ENABLE_ENTAB 0
/* 列データに「"」を含める場合「\"」とする。 = 1 */
#define ENABLE_ENQUOT 0
/* 列データに「"」を含める場合「""」とする。 = 1 */
#define ENABLE_QUOTQUOT 1
#define MODE_CSV 0x0001
#define MODE_TSV 0x0002
#define MODE_FIXROWS 0x0040
#define STATE_CSV_WAIT_QUOT 0
#define STATE_CSV_NEXT_QUOT 1
#define STATE_CSV_NEXT_TAB 2
#define STATE_CSV_DONE 3
#define STATE_CSV_BUFFEROVER 4
static char *_state[] = {
"STATE_CSV_WAIT_QUOT ",
"STATE_CSV_NEXT_QUOT ",
"STATE_CSV_NEXT_TAB ",
"STATE_CSV_DONE ",
"STATE_CSV_BUFFEROVER ",
};
/* ------------------------------------------------------------------ */
struct csv_work
{
int state;
int mode;
int rows;
char **x; /* テンポラリポインタ
- read_csv で渡したバッファを解放しない場合利用可能 */
char **node; /* 要素 */
int node_pos; /* 現在の要素数 */
int node_max; /* 要素最大サイズ */
};
/* ------------------------------------------------------------------ */
void csv_free( struct csv_work* csv )
{
int i;
for( i = 0; i < csv->node_max; i++ )
if(csv->node[i]) free(csv->node[i]);
free( csv->x );
free( csv );
}
/* ------------------------------------------------------------------ */
struct csv_work* csv_init( int node )
{
int i;
unsigned int x_buffsize;
struct csv_work* csv;
csv = (struct csv_work*)malloc( sizeof(struct csv_work) );
csv->state = STATE_CSV_WAIT_QUOT;
csv->mode = 0;
csv->rows = 1;
x_buffsize = sizeof(char*) * node;
csv->node_max = node;
csv->node_pos = 0;
csv->x = (char**)malloc(x_buffsize);
csv->node = (char**)malloc(x_buffsize);
for( i = 0; i < node; i++ )
csv->node[i] = NULL;
return csv;
}
/* ------------------------------------------------------------------ */
void csv_read_cat( struct csv_work* csv, char * p )
{
if(csv->node[csv->node_pos] == NULL)
csv->node[csv->node_pos] = strdup(csv->x[csv->node_pos]);
/* 既にdup されている場合、前回途中で止まっている */
/* 前回の分と連結 */
else {
char *cat;
cat = (char *)malloc(( strlen(p) +
strlen(csv->node[csv->node_pos]) + 1) * sizeof(char) );
strcpy( cat, csv->node[csv->node_pos] );
strcat( cat, p );
free(csv->node[csv->node_pos]);
csv->node[csv->node_pos] = cat;
}
return ;
}
/* ------------------------------------------------------------------ */
void csv_read( struct csv_work* csv, char * buff )
{
int i, before;
char *p;
char *q;
p = buff;
before = csv->node_pos;
for(;*p != '\0';) {
// printf("%s[%x](%d/%d):%s\n", _state[csv->state], csv->mode, csv->node_pos, csv->rows, p);
switch(csv->state)
{
case STATE_CSV_WAIT_QUOT:
/* skip space */
for (q = p;*q != '\0';q++) {
if(*q != ' ') break;
}
/* quote? */
if ( *q != '\"' ) {
/* p = space とばす前 */
/* カンマまたは タブを捜す*/
csv->state = STATE_CSV_NEXT_TAB;
}
else {
p = q;
/* " " 間で囲まれている */
p++; /* " を飛ばす */
csv->state = STATE_CSV_NEXT_QUOT;
}
if( csv->node_pos > csv->node_max )
csv->state = STATE_CSV_BUFFEROVER;
csv->x[csv->node_pos] = p;
break;
/* 次の " まで移動 */
case STATE_CSV_NEXT_QUOT:
for (q = p;*q != '\0';q++) {
if (*q == '\"') {
#if ENABLE_QUOTQUOT
if(
/* "" は無視する */
( *(q + 1) == '\"' )) {
q = q + 1;
continue;
}
#endif /* ENABLE_QUOTQUOT */
#if ENABLE_ENQUOT
if (( q != p ) &&
/* \" は無視する */
( *(q - 1) == '\\' )) {
continue;
}
#endif /* ENABLE_ENQUOT */
*q = '\0';
/* 次のノードに移動する */
q++;
csv_read_cat( csv, p );
p = q;
csv->state = STATE_CSV_NEXT_TAB;
break;
}
}
if( *q == '\0' ) {
csv_read_cat( csv, p );
p = q;
}
break;
/* タブまたはカンマ区切りを捜す */
case STATE_CSV_NEXT_TAB:
for (q = p;*q != '\0';q++) {
if ((*q == ',') ||
/* 既にCSVとして読み込んでいるのであれば、
TABはそのまま取り込みます。 */
((*q == '\t') && (!(csv->mode & MODE_CSV))) ||
(*q == '\r') || (*q == '\n')) {
#if ENABLE_ENTAB
/* 先頭で区切り発見 */
if( ( q == p ) ||
/* \, \\t は無視する */
( *(q - 1) != '\\' ) ) {
#endif /* ENABLE_ENTAB */
if(*q == ',') csv->mode |= MODE_CSV;
if(*q == '\t') csv->mode |= MODE_TSV;
/* ノード数が決定しました */
if(*q == '\r' || *q == '\n') {
csv->mode |= MODE_FIXROWS;
if( *(q +1) == '\r' || *(q +1) == '\n' )
*q++ = '\0';
}
/* 区切り文字によってノード数を推測します */
if(!(csv->mode & MODE_FIXROWS)) csv->rows++;
*q++ = '\0';
p = q;
csv->state = STATE_CSV_DONE;
break;
#if ENABLE_ENTAB
}
#endif /* ENABLE_ENTAB */
}
}
if( *q == '\0' ) {
csv_read_cat( csv, p );
p = q;
}
break;
case STATE_CSV_DONE:
/* printf( "DONE. %d: %s\n", csv->node_pos, csv->x[csv->node_pos]);*/
if(csv->node[csv->node_pos] == NULL)
csv->node[csv->node_pos] = strdup(csv->x[csv->node_pos]);
csv->node_pos++;
csv->state = STATE_CSV_WAIT_QUOT;
break;
case STATE_CSV_BUFFEROVER:
perror("buffer over!");
break;
}
}
return ;
}
/* ------------------------------------------------------------------ */
void csv_print( struct csv_work* csv, int row )
{
int i;
if(row == -1)
row = csv->rows;
for( i = 0; i <= csv->node_pos; i++ )
{
printf( "%d => %s\n", i % row + 1, csv->node[i]);
/*
* printf( "要素 %d行%d桁: %s\n", i / row + 1, i % row + 1, csv->node[i]);
*/
if( (i+1) % (row) == 0)
printf( "\n");
}
return ;
}
/* ------------------------------------------------------------------ */
/* 辻褄 */
void csv_fix( struct csv_work* csv )
{
char *p, *q;
int i;
for( i = 0; i <= csv->node_pos; i++ ) {
/* 5. 列データに「"」を含める場合「""」とする。 */
p = csv->node[i];
q = csv->node[i];
for( q = p ; *p != '\0';) {
#if ENABLE_QUOTQUOT
if( *q == '\"' && *(q +1) == '\"') q++;
#endif /* ENABLE_QUOTQUOT */
#if ENABLE_ENQUOT
else if( *q == '\\' && *(q +1) == '\"') q++;
#endif /* ENABLE_ENQUOT */
*p++ = *q++;
}
}
return ;
}
/* ------------------------------------------------------------------ */
int main()
{
char buff[256];
int len;
int before_pos;
struct csv_work * csv;
csv = csv_init( 400 );
/* fgets: \n か 256 読み込んだ文字列の最後に、\0を付加します */
while (fgets(buff, 256 - 1, stdin) != NULL) {
csv_read( csv, buff );
}
csv_fix( csv );
csv_print( csv, -1 );
csv_free( csv );
return 0;
}
/**
------------= csv.txt =------------------
"aaa","b
bb","ccc",zzz,"y""Y""y",xxx
-----------------------------------------
csv>gcc main.c
csv>a.exe < csv.txt
1 => aaa
2 => b
bb
3 => ccc
4 => zzz
5 => y"Y"y
6 => xxx
-----------------------------------------
**/
|
csv->node の解放が抜けていました。
1 2 3 4 5 6 7 8 9 10 | void csv_free( struct csv_work* csv )
{
int i;
for( i = 0; i < csv->node_max; i++ )
if(csv->node[i]) free(csv->node[i]);
free( csv->node );
free( csv->x );
free( csv );
}
|
変態的と名高い(?) Boost.Spirit で解析。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | #include <vector>
#include <string>
#include <exception>
#include <stdexcept>
#include <boost/spirit.hpp>
#include <boost/spirit/actor/push_back_actor.hpp>
#include <boost/spirit/actor/clear_actor.hpp>
typedef std::vector<std::string> csv_elem_t;
std::vector<csv_elem_t>
parse_csv(
std::string lines
)
{
using namespace boost::spirit;
std::vector<csv_elem_t> csv;
csv_elem_t e;
rule<> element_r = *((anychar_p - ch_p('"')) | str_p("\"\""));
rule<> quoted_r = ch_p('"') >> element_r[push_back_a(e)] >> ch_p('"');
rule<> naked_r = (*(anychar_p - ch_p('"') - ch_p(',') - eol_p))[push_back_a(e)];
rule<> record_r = list_p((quoted_r|naked_r), ch_p(','));
rule<> csv_r = list_p(record_r[push_back_a(csv,e)][clear_a(e)], eol_p) >> end_p;
parse_info<> result = parse(lines.c_str(), csv_r);
if ( !result.full ) {
throw std::runtime_error("failed to parse");
}
typedef std::vector<csv_elem_t>::iterator csv_list_iter;
typedef csv_elem_t::iterator csv_iter;
for ( csv_list_iter clit = csv.begin(); clit != csv.end(); ++clit ) {
for ( csv_iter cit = clit->begin(); cit != clit->end(); ++cit ) {
std::string::size_type idx=0;
while ( (idx = cit->find("\"\"", idx)) != std::string::npos ) {
cit->replace(idx, 2, "\""); ++idx;
}
}
}
return csv;
}
int main()
{
try {
std::vector<csv_elem_t> csv =
parse_csv("\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx");
std::cout << "total records: " << csv.size() << "\n";
typedef std::vector<csv_elem_t>::const_iterator csv_list_iter;
typedef csv_elem_t::const_iterator csv_iter;
int l = 1;
for ( csv_list_iter clit = csv.begin(); clit != csv.end(); ++clit,++l ) {
std::cout << "#" << l << "\n";
int i = 1;
for ( csv_iter cit = clit->begin(); cit != clit->end(); ++cit,++i ) {
std::cout << i << " => " << *cit << "\n";
}
}
}
catch ( std::exception& e ) {
std::cerr << e.what() << "\n";
}
return 0;
}
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | (* ocaml camlp4rf.cma もしくは
ocamlc -pp "camlp4rf" a.ml *)
value add_opened_csv_string =
let rec loop buf = parser
[ [: `'"'; st:] ->
if (Stream.peek st <> Some '"') then ()
else (Buffer.add_char buf (Stream.next st); loop buf st)
| [: `c ; st :] -> (Buffer.add_char buf c; loop buf st) ]
in fun buf st -> loop buf st;
value record_iteri =
let use_buffer f buf =
(f (Buffer.contents buf); Buffer.clear buf) in
let rec loop f pos buf = parser
[ [: `','; st:] -> (use_buffer (f pos) buf; loop f (pos+1) buf st)
| [: `'\n'; st:] -> use_buffer (f pos) buf
| [: `'"'; st:] -> (add_opened_csv_string buf st; loop f pos buf st)
| [: `c ; st:] -> (Buffer.add_char buf c; loop f pos buf st)
| [: :] -> use_buffer (f pos) buf ]
in fun st f buf -> loop f 1 buf st;
(*
value t = Stream.of_string "\
\"aaa\",\"b\n\
bb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx";
record_iteri t (Printf.printf "%2d => %s\n") (Buffer.create 8);
*)
|
なでしこでは「CSV取得」命令で配列に変換できます。
1 2 3 4 5 | 「"aaa","b
bb","ccc",zzz,"y""Y""y",xxx」をCSV取得
反復
反復
回数＆「 => 」＆対象を表示
|
標準入力→標準出力です。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | function splitCSV() {
local c in_dq after_dq
local -i count=0
while read -n 1 c; do
if [ -z "$in_req" ]; then
in_req=1
echo -n "$((++count)) => "
fi
: ${c:=$'\n'} # 改行は空文字として読まれる
if [ "$c" = \" ]; then
if [ -n "$after_dq" ]; then
echo -n \"
after_dq=''
else
after_dq=1
fi
else
if [ -n "$after_dq" ]; then
after_dq=''
if [ -n "$in_dq" ]; then
in_dq=''
else
in_dq=1
fi
fi
if [ -z "$in_dq" -a \( "$c" = , -o "$c" = $'\n' \) ]; then
in_req=''
echo
else
echo -n "$c"
fi
fi
done
}
|
正規表現等で頑張ってみました。 すっきり書けたかなと思います。 [実行結果] 1 => aaa 2 => b bb 3 => ccc 4 => zzz 5 => y"Y"y 6 => xxx
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | def csv = '''\
"aaa","b
bb","ccc",zzz,"y""Y""y",xxx\
'''
resolveCSV(csv).eachWithIndex{ it, idx -> println "${idx+1} => ${it}" }
/** CSVレコードの分解(RFC 4180対応版) */
def resolveCSV(String csv) {
csv.split(',').inject(['""']){ result, it ->
// 一個前が「半開」ならそこに追加
if (result[-1] ==~ /^".*[^"]$/ || result[-1].count('"') % 2 == 1) {
result[-1] = [result[-1], it].join(',') // 「,」で結合
// 一個前が「閉」なら新しく要素を追加
} else {
// 「"」で囲んでおく
result << ((it =~ /"/) ? it : '""'.toList().join(it))
}
result
}[1..-1]*.replaceAll(/^"|"$/, '')*.replaceAll(/"{2}/, '"')
}
|
opencsvライブラリを使用しました。 (opencsv - an open source csv parser for Java <http://opencsv.sourceforge.net/>)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 | import au.com.bytecode.opencsv.*
import java.io.*
def csv = '''\
"aaa","b
bb","ccc",zzz,"y""Y""y",xxx\
'''
new CSVReader(new StringReader(csv)).readAll().each{
def i = 1
it.each{
println "${i++} => ${it}"
}
}
|
オブジェクト指向らしく(?)STATEパターン的なものを作りました。 CallBackにはレコード/CSVの終了がプッシュされてきますがこのお題では使ってません。 改行コードはReaderが全て\nに変換してくれるので、どのタイプでも大丈夫なはずです。 個人的にはappend(), collectToken(), endOfParse() あたりをもう少しなんとか…とか 各Handler#handle()がどうも似てるような気がしてどうにかならないものかと…。 実行例： $java CSVParser 1 => aaa 2 => b bb 3 => ccc 4 => zzz 5 => y"Y"y 6 => xxx 7 => eee,EEE 8 => 9 => 10 => 11 => 12 => DDD 13 => 14 => fff
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 | import java.io.IOException;
import java.io.LineNumberReader;
import java.io.Reader;
import java.io.StringReader;
public class CSVParser {
public static void main(String[] args) throws IOException {
CSVParser parser = new CSVParser(new CallBack() {
private int num = 0;
@Override
public void setToken(String token) {
System.out.println((++num)+" => "+token);
}
@Override
public void endRecord() {}
@Override
public void endCsv() {}
});
parser.parse(new StringReader("\"aaa\",\"b\nbb\",\"ccc\",zzz,\"y\"\"Y\"\"y\",xxx,\"eee,EEE\",,,\n,DDD,\"\",fff"));
}
public interface CallBack {
/**
* CSVのトークンを切り出した後に、格納するためにパーサから呼び出される。
* @param token CSVトークン
*/
void setToken(String token);
/**
* CSVレコードの最後のトークンの setToken() を呼び出した後にパーサから呼び出される。
*/
void endRecord();
/**
* 最後のCSVレコードのendRecord() を呼び出した後にパーサから呼び出される。
*/
void endCsv();
}
/**
* CSVParserが処理中に発生した例外を示す実行時例外クラス
* <p>処理中だったCSVファイルの行番号を格納する。
*/
public class ParseException extends RuntimeException {
private static final long serialVersionUID = 1L;
private int linenumber;
ParseException(int linenumber) {
super();
this.linenumber = linenumber;
}
ParseException(String message, int linenumber) {
super(message);
this.linenumber = linenumber;
}
ParseException(String message, int linenumber, Throwable cause) {
super(message, cause);
this.linenumber = linenumber;
}
ParseException(int linenumber, Throwable cause) {
super(cause);
this.linenumber = linenumber;
}
/**
* CSVファイルの行番号を返す
* @return 行番号(1～)
*/
public int getLineNumber() { return linenumber; }
/**
* 詳細メッセージを返します。
* @return 詳細メッセージ(無い場合はnull)
*/
@Override
public String getMessage() {
return super.getMessage()+" [line="+linenumber+"]";
}
}
private LineNumberReader lnreader;
private CallBack callback;
private StringBuffer tokenBuffer = new StringBuffer();
public CSVParser(CallBack callback) {
if(callback == null) throw new NullPointerException();
this.callback = callback;
}
public void parse(Reader reader) throws IOException {
lnreader = new LineNumberReader(reader);
for(Handler h=firstRecordCharHandler; h!=null; h=h.handle(lnreader.read()));
}
protected boolean isComment(int c) { return c == '#'; }
protected boolean isQuot(int c) { return c == '"'; }
protected boolean isEndOfToken(int c) { return c == ',' || isEndOfRecord(c); }
protected boolean isEndOfRecord(int c) { return c == '\n' || isEndOfCsv(c); }
protected boolean isEndOfCsv(int c) { return c == -1; }
protected Handler append(int c, Handler next) {
tokenBuffer.append((char)c);
return next;
}
protected Handler collectToken(int c) {
callback.setToken(tokenBuffer.toString());
tokenBuffer.setLength(0);
if(isEndOfRecord(c)) {
callback.endRecord();
if(isEndOfCsv(c)) return endOfParse();
return firstRecordCharHandler;
}
return firstTokenCharHandler;
}
protected Handler endOfParse() {
callback.endCsv();
return null;
}
private abstract class Handler {
abstract Handler handle(int c);
}
private final Handler firstRecordCharHandler = new FirstRecordCharHandler();
private final Handler skipLineHandler = new SkipLineHandler();
private final Handler firstTokenCharHandler = new FirstTokenCharHandler();
private final Handler quottingTokenHandler = new QuottingTokenHandler();
private final Handler quotInQuotCharHandler = new QuotInQuotCharHandler();
private final Handler unquottingTokenHandler = new UnquottingTokenHandler();
/**
* CSVのレコード（行）の最初の文字を処理するハンドラクラス
* <p>
* '"'で始まる場合はコメント行として改行までスキップし、
* それ以外の（有効）文字はCSVトークンの始まりとして処理し、
* EOFなら解析を終了する。
*
* @see CSVParser#isComment(int)
* @see CSVParser#isEndOfCsv(int)
* @see CSVParser#endOfParse(int)
*/
private class FirstRecordCharHandler extends Handler {
@Override
public Handler handle(int c) {
if(isComment(c)) return skipLineHandler;
if(isEndOfCsv(c)) return endOfParse();
return firstTokenCharHandler.handle(c);
}
}
/**
* CSVの一行を読み飛ばすハンドラクラス
* <p>
* EOFなら解析を終了し、改行なら firstRecordCharHandler に以降の処理を任せる。
*/
private class SkipLineHandler extends Handler {
@Override
public Handler handle(int c) {
if(isEndOfCsv(c)) return endOfParse();
if(isEndOfRecord(c)) return firstRecordCharHandler;
return this;
}
}
/**
* CSVトークンの最初の文字を処理するハンドラクラス
* <p>
* '"'なら次の'"'までをトークンとして扱い、それ以外の（有効）文字なら
* トークン終了文字（','、改行、EOF）までをトークンとして扱うようにする。
*
* @see CSVParser#isQuot(int)
* @see CSVParser.QuottingTokenHandler
* @see CSVParser.UnquottingTokenHandler
*/
private class FirstTokenCharHandler extends Handler {
@Override
public Handler handle(int c) {
if(isQuot(c)) return quottingTokenHandler;
return unquottingTokenHandler.handle(c);
}
}
/**
* '"'で囲まれたCSVトークンを取り出すハンドラクラス
* <p>
* '"'が現れるまで文字格納処理を行い、現れたらエスケープ判定を行うハンドラクラスに委譲する。
*
* @throws ParseException 終了を示す'"'が現れる前にEOFになった場合
*
* @see CSVParser#isQuot(int)
* @see CSVParser#isEndOfCsv(int)
* @see CSVParser.QuotInQuotCharHandler
*/
private class QuottingTokenHandler extends Handler {
@Override
public Handler handle(int c) {
if(isQuot(c)) return quotInQuotCharHandler;
if(isEndOfCsv(c)) throw new ParseException("quot error.", lnreader.getLineNumber()+1);
return append(c, this);
}
}
/**
* '"'で始まったCSVトークン内で'"'が表れた場合のハンドラクラス
* <p>
* 続いて'"'が来ればエスケープされた'"'と見なして文字格納処理を行い、
* トークン終了文字が来ればトークン終了と見なしてトークン格納処理を行う。
*
* @throws ParseException '"'もしくはトークン終了文字以外が表れた場合
*
* @see CSVParser#isQuot(int)
* @see CSVParser#isEndOfToken(int)
* @see CSVParser.QuottingTokenHandler
*/
private class QuotInQuotCharHandler extends Handler {
@Override
public Handler handle(int c) {
if(isEndOfToken(c)) return collectToken(c);
if(isQuot(c)) return append(c, quottingTokenHandler);
throw new ParseException("quot error.", lnreader.getLineNumber()+1);
}
}
/**
* '"'で囲まれていないCSVトークンを取り出すハンドラクラス
* <p>
* トークン終了文字が表れるまで文字格納処理を行い、表れたらトークン格納処理を行う。
*
* @throws ParseException '"'が表れた場合
*
* @see CSVParser#isQuot(int)
* @see CSVParser#isEndOfToken(int)
* @see CSVParser#collectToken(int)
*/
private class UnquottingTokenHandler extends Handler {
@Override
public Handler handle(int c) {
if(isEndOfToken(c)) return collectToken(c);
if(isQuot(c)) throw new ParseException("quot error.", lnreader.getLineNumber()+1);
return append(c, this);
}
}
}
|
.NET Framework 2.0 以降でのみ使用可能です。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | Imports Microsoft.VisualBasic.FileIO
Module CSVParser
Sub Main()
Dim p As New TextFieldParser(Console.OpenStandardInput())
p.SetDelimiters(",")
p.HasFieldsEnclosedInQuotes = True
While Not p.EndOfData
Dim i As Integer = 0
For Each col As String In p.ReadFields()
i = i + 1
Console.WriteLine("{0} => {1}", i, col)
Next col
End While
End Sub
End Module
|
C#でStateパターン風に実装。
.NET FrameWork 2.0 以上だと #9257 の Microsoft.VisualBasic.FileIO を使えばいいわけなので、1.1 の言語機能だけで実現するべくジェネリックとか Linq とかは使わないようにしてみました。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 | using System;
using System.Collections;
using System.Text;
using System.IO;
class ParserState
{
private TextFieldParser _p;
private bool _break = true;
private int _openQ = 0;
private bool _isClosing = false;
private StringBuilder _curFldDlm;
private StringBuilder _curRecTrm;
public bool IsBreaking { get { return _break; } }
public bool IsQuoted { get { return (_openQ != 0); } }
public bool IsClosing { get { return _isClosing; } }
public bool IsFieldDelimited { get { return Grep(_curFldDlm.ToString(), _p.FieldDelimiters); } }
public bool IsRecordTerminated { get { return Grep(_curRecTrm.ToString(), _p.RecordTerminaters); } }
private bool Grep(string s, string[] c)
{
foreach (string t in c)
if (s == t) return true;
return false;
}
private bool CheckIfQuote(int input)
{
foreach (int c in _p.Quoters)
if (input == c) return true;
return false;
}
private bool CheckIfFieldEnding(int input)
{
foreach (string s in _p.FieldDelimiters)
if (s.StartsWith(_curFldDlm.ToString() + ((char)input).ToString()))
return true;
return false;
}
private bool CheckIfRecordEnding(int input)
{
foreach (string s in _p.RecordTerminaters)
if (s.StartsWith(_curRecTrm.ToString() + ((char)input).ToString()))
return true;
return false;
}
private bool IsTransit { get { return _curFldDlm.Length > 0 || _curRecTrm.Length > 0; } }
public ParserState(TextFieldParser parser)
{
_p = parser;
}
public int GetChar(int input, out string backtrack)
{
backtrack = String.Empty;
if (_break)
{
_curFldDlm = new StringBuilder(_p.MaxFieldDelimiterLength);
_curRecTrm = new StringBuilder(_p.MaxRecordTerminaterLength);
if (CheckIfQuote(input))
{
_break = false;
_openQ = input;
_isClosing = false;
return -1;
}
}
if (IsQuoted)
{
if (input == _openQ)
{
_isClosing = !_isClosing;
return _isClosing ? -1 : input;
}
if (!IsClosing) return input;
//after the quote has been closed
_openQ = 0;
_isClosing = false;
}
if (CheckIfFieldEnding(input))
_curFldDlm.Append((char)input);
else if (_curFldDlm.Length > 0)
{
backtrack = _curFldDlm.ToString();
_curFldDlm = new StringBuilder(_p.MaxFieldDelimiterLength);
}
if (CheckIfRecordEnding(input))
_curRecTrm.Append((char)input);
else if (_curRecTrm.Length > 0)
{
backtrack = _curRecTrm.ToString();
_curRecTrm = new StringBuilder(_p.MaxRecordTerminaterLength);
}
_break = this.IsFieldDelimited || this.IsRecordTerminated;
return this.IsTransit ? -1 : input;
}
}
class TextFieldParser
{
private readonly StreamReader _reader;
private readonly char[] _q;
private readonly string[] _fldDlm;
private readonly string[] _rcdTrm;
private readonly int _fldDlmLen = 0;
private readonly int _rcdTrmLen = 0;
private readonly bool _trmSpc = false;
public TextFieldParser(
Stream stream,
char[] quoters,
string[] fieldDelimiters,
string[] recordTerminaters,
bool trimSpaces)
{
_reader = new StreamReader(stream);
_q = quoters;
_fldDlm = fieldDelimiters;
foreach (string s in _fldDlm)
if (s.Length > _fldDlmLen)
_fldDlmLen = s.Length;
_rcdTrm = recordTerminaters;
foreach (string s in _rcdTrm)
if (s.Length > _rcdTrmLen)
_rcdTrmLen = s.Length;
_trmSpc = trimSpaces;
}
public TextFieldParser(Stream stream, char[] quoters,
string[] fieldDelimiters, string[] recordTerminaters) :
this(stream, quoters, fieldDelimiters, recordTerminaters, false) { }
public TextFieldParser(Stream stream, char[] quoters, string[] fieldDelimiters) :
this(stream, quoters, fieldDelimiters, new string[]{ "\r\n" }) { }
public TextFieldParser(Stream stream, char[] quoters) :
this(stream, quoters, new string[] { "," }) { }
public TextFieldParser(Stream stream) :
this(stream, new char[] { '"' }) { }
public char[] Quoters { get { return (char[])_q.Clone(); } }
public string[] FieldDelimiters { get { return (string[])_fldDlm.Clone(); } }
internal int MaxFieldDelimiterLength { get { return _fldDlmLen; } }
public string[] RecordTerminaters { get { return (string[])_rcdTrm.Clone(); } }
internal int MaxRecordTerminaterLength { get { return _rcdTrmLen; } }
public bool EndOfData { get { return _reader.EndOfStream; } }
int _fldCnt = 0;
public string[] ReadFields()
{
if (_reader.Peek() == -1) return null;
ParserState stat = new ParserState( this );
ArrayList fields = new ArrayList( _fldCnt );
StringBuilder field = new StringBuilder();
int nextChr;
while (-1 != (nextChr = _reader.Read()))
{
string buf;
int ret = stat.GetChar(nextChr, out buf);
if (ret != -1) field.Append(buf).Append((char)ret);
if (stat.IsBreaking)
{
string f = field.ToString();
if (_trmSpc) f = f.Trim();
fields.Add(f);
field = new StringBuilder();
}
if (stat.IsRecordTerminated) break;
}
if (_fldCnt < fields.Count) _fldCnt = fields.Count;
return (string[])fields.ToArray(typeof(string));
}
public string[][] Parse()
{
ArrayList records = new ArrayList();
while (!this.EndOfData)
records.Add(this.ReadFields());
return (string[][])records.ToArray();
}
}
class Program
{
static void Main(string[] args)
{
TextFieldParser p = new TextFieldParser(Console.OpenStandardInput());
while (!p.EndOfData)
{
int i = 0;
foreach (string col in p.ReadFields())
Console.WriteLine("{0} => {1}", ++i, col);
}
}
}
|
VBScriptはテキスト加工に適した言語なのですが、 まだ投稿がなかったのでやってみました。 C#版の単純移植なので、あまり綺麗なコードではありません。。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | s = """aaa"",""b" & vbCrLf & "bb"",""ccc"",zzz,""y""""Y""""y"",xxx"
ss = splitCSV(s)
i = 1
For Each elem In ss
WScript.Echo i & " => " & elem
i = i + 1
Next
Function splitCSV(s)
If IsNull(s) Then
splitCSV = Null
Exit Function
End If
Dim a()
index = 0
result = ""
i = 1
While i <= Len(s)
b = (mid(s, i, 1) = """")
If b Then
i = i + 1
End If
If b Then
j = InStr(i, s, """")
Else
j = InStr(i, s, ",")
End If
If j < 1 Then
j = Len(s) + 1
End If
t = mid(s, i, j - i)
If b And (j < Len(s) - 1) And Mid(s, j+1, 1) = """" Then
result = result & t
result = result & """"
i = j + 1
Else
ReDim Preserve a(index)
a(index) = result & t
index = index + 1
result = ""
If b Then
i = j + 2
Else
i = j + 1
End If
End If
Wend
splitCSV = a
End Function
|
raynstard
#3389()
Rating1/1=1.00
[ reply ]