DouKaku?

import os, sys, dumbdbm, md5

db = dumbdbm.open('tmp')
fp = file(sys.argv[1], 'r')
i = 0
for s in fp:
  k = md5.md5(s).digest()
  if db.has_key(k):
    db[db[k]] = '0'
  db[k] = str(i)
  db[str(i)] = '1'
  i += 1

fp.seek(0)
i = 0
for s in fp:
  if db[str(i)] == '1':
    sys.stdout.write(s)
  i += 1
fp.close()
db.close()

os.remove('tmp.dat')
os.remove('tmp.dir')
os.remove('tmp.bak')

(define (main args)
  (call-with-input-file (cadr args)
    (lambda (port)
      (let loop ((next 0))
        (port-seek port next)
        (let ((buf (read-line port))
              (next (port-tell port)))
          (unless (eof-object? buf)
            (let loop1 ()
              (let1 bufn (read-line port)
                (cond ((eof-object? bufn) (print buf))
                      ((not (string=? bufn buf)) (loop1)))))
            (loop next)))))))

def fetch(fin, head)
  cur = fin.pos
  fin.pos = head
  s = fin.gets
  fin.pos = cur
  s
end

def line_with_headpos(fin)
  head = fin.pos
  while line = fin.gets
    yield(line, head)
    head = fin.pos
  end
end

def zoku(fn, n=5)
  open(fn){|fin|
    h = {}

    line_with_headpos(fin){|line, head|
      (h[line[0,n]] ||= []) << head
    }

    fin.rewind

    line_with_headpos(fin){|line, head|
      flag = h[line[0,n]].any?{|ps|
        head < ps && line == fetch(fin, ps)
      }
      print line unless flag
    }
  }
end

zoku(ARGV[0])

#include <stdio.h>
#include <fcntl.h>
#include <sys/types.h>
#include <unistd.h>

char same(int fin, off_t a, off_t b){
    char x[1],y[1];
    char ret;

    while (1){
        lseek(fin, a++, SEEK_SET);
        ret = read(fin, x, 1);

        lseek(fin, b++, SEEK_SET);
        ret += read(fin, y, 1);

        if (ret == 1 || x[0] != y[0]){
            return 0;
        }
        if (ret == 0 || x[0] == '\n'){
            return 1;
        }
    }
}

off_t next_line(int fin, off_t a){
    char x[1];
    char ret;

    lseek(fin, a, SEEK_SET);
    do{
        ret = read(fin, x, 1);
    }while(x[0] != '\n' && ret > 0);

    return ret == 0 ? 0 : lseek(fin, 0, SEEK_CUR);
}

void print(int fin, off_t a){
    char x[1];
    char ret;

    lseek(fin, a, SEEK_SET);
    do{
        ret = read(fin, x, 1);
        if (ret == 0)
            break;

        printf("%c", x[0]);
    }while(x[0] != '\n');
}

void zoku(int fin){
    off_t a=0,b=0;

    do{
        char flag = 1;

        lseek(fin, a, SEEK_SET);
        b = a;
        while ((b = next_line(fin, b))){
            if (same(fin, a, b)){
                flag = 0;
                break;
            }
        }
        if (flag)
            print(fin ,a);

    }while((a = next_line(fin, a)));
}

int main(int argc, char** argv){
    int fin;

    if (argc <= 1)
        return 1;

    fin = open(argv[1], O_RDONLY);

    if (fin < 0)
        return 1;

    zoku(fin);

    return 0;
}

mysql -uroot -N -r test << EOF

DROP TABLE IF EXISTS tmp;
CREATE TABLE tmp (
  id INT PRIMARY KEY AUTO_INCREMENT,
  line LONGTEXT NOT NULL,
  INDEX (line(10))
) CHARACTER SET UTF8;

LOAD DATA LOCAL INFILE "$1" INTO TABLE tmp FIELDS TERMINATED BY '\n' (line);

SELECT line FROM tmp GROUP BY line ORDER BY max(id);

EOF

require 'tempfile'

def make_big_seq_file(cntmax,lthmin,eps)
  cntmax.times {
    s = ""
    (lthmin+rand(eps)).times {
      s << rand(10).to_s
    }
    puts s
  }
end

#make_big_seq_file(10**6,50,20)

def double_line_reject(f)
  h = Hash.new(0)

  temp = Tempfile::new("foobar")
  open(f) {|fin|
    while line = fin.gets
      temp.puts(line.hash)
    end
    temp.close

    f = open("|sort #{temp.path}")
    iold = ""
    f.each {|i|
      i = i.chomp.to_i
      h[i] += 1 if iold == i
      iold = i
    }
    fin.rewind
    while line = fin.gets
      puts line if h[line.hash] < 1
      h[line.hash] -= 1
    end
  }
end

double_line_reject(ARGV[0])

#include <string>
#include <iostream>
#include <fstream>
#include <list>
#include <vector>
#include <numeric>
#include <algorithm>
#include <functional>
using namespace std;

template <int SIZE>
struct StringHashFunc : public binary_function<int, unsigned char, int> {
    int operator()(int h, unsigned char c) const {
        return (h * 137 + c) % SIZE;
    }
};
template <class T, int SIZE>
class StringHashTable {
    vector<list<T> > table;
public:
    StringHashTable<T, SIZE>() : table(SIZE) {}
    list<T> &operator[](const string &s) {
        return table[accumulate(s.begin(), s.end(), 0, StringHashFunc<SIZE>())];
    }
};