DouKaku?

using System;
using System.Collections.Generic;
using System.Text;
using System.IO;

class Program
{
  static Encoding enc = Encoding.Default;
  static void Main(string[] args)
  {
    IDictionary<int, List<long>> hashtable = new Dictionary<int, List<long>>();
    SortedList<long, List<long>> collision = new SortedList<long, List<long>>();
    using (FileStream fs = new FileStream(args[0], FileMode.Open, FileAccess.Read))
    {
      using (BufferedStream bs = new BufferedStream(fs))
      {
        // 重複候補の位置をリストに保存
        long currentPosition = 0L;
        byte[] bytes = null;
        while ((bytes = ReadLine(bs)).Length > 0)
        {
          string line = enc.GetString(bytes);
          int hash = line.GetHashCode();
          List<long> list = null;
          if (!hashtable.ContainsKey(hash))
          {
            list = new List<long>();
            hashtable.Add(hash, list);
          }
          else
          {
            list = hashtable[hash];
            if (list.Count == 1)
              collision.Add(list[0], list);
            collision.Add(currentPosition, list);
          }
          list.Add(currentPosition);
          currentPosition += bytes.Length;
        }
        // 重複候補の値を調べ、末尾側に重複があるか検証
        currentPosition = 0L;
        bs.Position = currentPosition;
        for (int baseIndex = 0; baseIndex < collision.Count; baseIndex++)
        {
          long basePosition = collision.Keys[baseIndex];
          bs.Seek(basePosition - currentPosition, SeekOrigin.Current);
          currentPosition = basePosition;
          byte[] baseBytes = ReadLine(bs);
          currentPosition += baseBytes.Length;
          string baseString = enc.GetString(baseBytes);
          List<long> targetList = collision.Values[baseIndex];
          bool found = false;
          for (int targetIndex = 0; targetIndex < targetList.Count; targetIndex++)
          {
            long targetPosition = targetList[targetIndex];
            if (targetPosition <= basePosition)
              continue;
            bs.Seek(targetPosition - currentPosition, SeekOrigin.Current);
            currentPosition = targetPosition;
            byte[] targetBytes = ReadLine(bs);
            currentPosition += targetBytes.Length;
            string targetString = enc.GetString(targetBytes);
            found = baseString.Equals(targetString);
            if (found)
              break;
          }
          if (!found)
            collision.RemoveAt(baseIndex--);
        }
        // 出力
        currentPosition = 0L;
        bs.Position = currentPosition;
        while ((bytes = ReadLine(bs)).Length > 0)
        {
          if (!collision.ContainsKey(currentPosition))
            Console.Write(enc.GetString(bytes));
          currentPosition += bytes.Length;
        }
      }
    }
  }

  static byte[] ReadLine(Stream s)
  {
    byte[] bytes = enc.GetBytes(Environment.NewLine);
    List<byte> line = new List<byte>();
    int b = 0;
    while ((b = s.ReadByte()) >= 0)
    {
      line.Add((byte)b);
      if (line.Count >= bytes.Length)
      {
        bool newline = false;
        for (int i = 0; i < bytes.Length; i++)
        {
          newline = (line[line.Count - bytes.Length + i] == bytes[i]);
          if (!newline)
            break;
        }
        if (newline)
          return line.ToArray();
      }
    }
    return line.ToArray();
  }
}