FileIndexer.cpp

Go to the documentation of this file.

/*

Copyright (C) 2005-2007 by Peter Dimov.

This file is part of Calitko (http://www.calitko.org).

Calitko is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

Calitko is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Calitko; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

*/

#include "Qt.h"
#include "FileIndexer.h"
#include "FileInfo.h"
//#include "Gnutella/PacketProcessing/QueryRoutingFilter.h"

using Gnutella::Searching::FileIndexer;
using Gnutella::Searching::FileInfo;
//using Gnutella::PacketProcessing::QueryRoutingFilter;

#define REFRESH_INTERVAL_MIN        100

QDataStream & operator<< (QDataStream &stream, QMap<uint, FileInfo*> &files)
{
    using namespace Gnutella::Searching;

    QList<uint> fileIndexes = files.keys();

    stream << fileIndexes.count();

    foreach (uint fileIndex, fileIndexes) {
        stream << fileIndex << *(files.value (fileIndex));
    }
    return stream;
}

QDataStream & operator>> (QDataStream &stream, QMap<uint, FileInfo*> &files)
{
    using namespace Gnutella::Searching;

    uint fileIndex;
    int numEntries;

    stream >> numEntries;


    for (int i=0; i<numEntries; i++) {
        FileInfo *fileInfo = new FileInfo();
        stream >> fileIndex >> *fileInfo;
//      qDebug() << fileInfo->fileName();
//      qDebug() << fileInfo->fileIndex();
        files.insert (fileIndex, fileInfo);
    }
    return stream;
}

QDataStream & operator<< (QDataStream &stream, QMap<QString, FileInfo*> &fileWords)
{
    QList<QString> words = fileWords.keys().toSet().toList();       // take the words without repetitions
    QList<FileInfo*> fileInfos;

    stream << words.count();

    foreach (QString word, words) {
        fileInfos = fileWords.values (word);

        stream << word;
        stream << fileInfos.count();

        foreach (FileInfo* fileInfo, fileInfos) {
            stream << fileInfo->fileIndex();
//          qDebug() << fileInfo->fileName();
//          qDebug() << fileInfo->fileIndex();
        }
    }

    return stream;
}

QDataStream& FileIndexer::readFileWords (QDataStream &stream)
{
    int numWords;
    stream >> numWords;

    int numInfos;
    uint fileIndex;
    QString word;
    for (int i=0; i<numWords; i++) {
        stream >> word;
        stream >> numInfos;

//      qDebug() << word;
        //p.queryRoutingFilter->addToQueryRoutingTable (word);

        for (int j=0; j<numInfos; j++) {
            stream >> fileIndex;
            Q_ASSERT (p.files.contains (fileIndex));
            p.fileWords.insertMulti (word, p.files.value (fileIndex));

//          qDebug() << p.files.value (fileIndex)->fileName();
        }
    }

    return stream;
}


QDataStream & operator<< (QDataStream &stream, QList<QDir> &dirs)
{
    stream << dirs.count();

    foreach (QDir dir, dirs) {
        stream << dir.absolutePath();
    }

    return stream;
}

QDataStream & operator>> (QDataStream &stream, QList<QDir> &dirs)
{
    int numDirs;
    stream >> numDirs;

    QString dirPath;
    for (int i=0; i<numDirs; i++) {
        stream >> dirPath;
//      qDebug() << dirPath;
        dirs.append (QDir (dirPath));
    }

    return stream;
}

FileIndexer::FileIndexer (/*QueryRoutingFilter *queryRoutingFilter*/)
 : p()
{
    p.checkThread = new CheckIndexThread (this);

    p.refreshTimer = new QTimer (this);
    QObject::connect (p.refreshTimer, SIGNAL (timeout()), p.checkThread, SLOT(start()));
    p.refreshTimer->setInterval (REFRESH_INTERVAL_MIN * 60000);
    p.refreshTimer->start();
    //p.queryRoutingFilter = queryRoutingFilter;

    QFile file ("fileindex.dat");

    if (!file.open (QIODevice::ReadOnly))
        return;

    QDataStream stream (&file);

    stream >> p.files;
    stream >> p.sharedFolders;
    readFileWords (stream);

    p.checkThread->start();
    //refreshIndex();
}

FileIndexer::~FileIndexer()
{
    p.refreshTimer->stop();
    delete p.refreshTimer;
    // <sm_todo> write the index to a file

    QFile file ("fileindex.dat");

    if (!file.open (QIODevice::WriteOnly))
        return;

    QDataStream stream (&file);

    stream << p.files;
    stream << p.sharedFolders;
    stream << p.fileWords;

}

void FileIndexer::refreshIndex()
{
    QTime time;
    time.start();
    QList<uint> fileIndexes;

    foreach (QDir sharedDir, p.sharedFolders) {
        sharedDir.refresh();
        qDebug() << sharedDir.absolutePath();
        if (!QFileInfo(sharedDir.absolutePath()).exists()) {
            removeFolder (sharedDir.absolutePath());
            continue;
        }

        QList<QString> fileEntries = sharedDir.entryList (QDir::Files, QDir::Name);

        foreach (QString fileName, fileEntries) {
    //      qDebug() << sharedDir.absoluteFilePath (fileName);
            FileInfo *fileInfo = new FileInfo (sharedDir.absoluteFilePath (fileName));

            fileIndexes.append (fileInfo->fileIndex());

            if (!p.files.value (fileInfo->fileIndex())) {
                indexFile (fileInfo);
            } else {
                delete fileInfo;
            }

        }
    }

    QList<uint> filesToRemove = (p.files.keys().toSet().subtract (fileIndexes.toSet())).toList();

    foreach (uint fileIndex, filesToRemove) {
        removeFile (p.files.value (fileIndex));
    }

    checkForAddedFolder();
    qDebug() << "Refreshing file index finished!  " << time.elapsed() << " ms";
}

void FileIndexer::checkForAddedFolder()
{
    foreach (QDir sharedDir, p.sharedFolders) {
        QList<QString> dirEntries = sharedDir.entryList (QDir::Dirs | QDir::NoSymLinks, QDir::Name);

        qDebug() << sharedDir.absolutePath();
        Q_ASSERT (sharedDir.exists());
        dirEntries.pop_front();     // .
        dirEntries.pop_front();     // ..

        foreach (QString dir, dirEntries) {
            if (!p.sharedFolders.contains (QDir (sharedDir.absoluteFilePath (dir)))) {
                indexFolder (sharedDir.absoluteFilePath (dir));
            }
        }
    }
}

bool FileIndexer::indexFolder (QString folderPath)
{
    QDir sharedDir (folderPath);

    if (!sharedDir.exists()) {
        return false;
    }

    if (p.sharedFolders.contains (sharedDir)) {
        return true;
    }

    qDebug() <<sharedDir.absolutePath();
    p.sharedFolders.append (sharedDir);

    QList<QString> fileEntries = sharedDir.entryList (QDir::Files, QDir::Name);

    foreach (QString fileName, fileEntries) {
        FileInfo *fileInfo = new FileInfo (sharedDir.absoluteFilePath (fileName));
        indexFile (fileInfo);
    }

    QList<QString> dirEntries = sharedDir.entryList (QDir::Dirs | QDir::NoSymLinks, QDir::Name);

    dirEntries.pop_front();     // .
    dirEntries.pop_front();     // ..

    foreach (QString dirName, dirEntries) {
        indexFolder (sharedDir.absoluteFilePath (dirName));
    }

    return true;
}

void FileIndexer::indexFile (QString filePath)
{
    if (! QFile::exists (filePath)) {
        return;
    }

    FileInfo *fileInfo = new FileInfo (filePath);

    indexFile (fileInfo);
}

void FileIndexer::indexFile (FileInfo *fileInfo)
{
    Q_ASSERT (fileInfo->exists());

    p.files.insert (fileInfo->fileIndex(), fileInfo);

    QString fileName = QFileInfo (fileInfo->fileName()).fileName();
    QList<QString> fileNameWords = fileName.toLower().split (QRegExp ("(\\s+|-|_|\\W)+"));

    foreach (QString word, fileNameWords) {
        if (word.isEmpty()) {
            continue;
        }
        p.fileWords.insertMulti (word, fileInfo);
        //p.queryRoutingFilter->addToQueryRoutingTable (word);
    }
}

bool FileIndexer::removeFolder (QString folderPath)
{
    QDir sharedDirToBeRemoved (folderPath);

    p.sharedFolders.removeAt (p.sharedFolders.indexOf (sharedDirToBeRemoved));
    removeFolderFiles (folderPath);

    if (!sharedDirToBeRemoved.exists()) {
        return true;
    }

    QList<QString> dirEntries = sharedDirToBeRemoved.entryList (QDir::Dirs | QDir::NoSymLinks, QDir::Name);

    dirEntries.pop_front();
    dirEntries.pop_front();

    foreach (QString dirName, dirEntries) {
        removeFolder (sharedDirToBeRemoved.absoluteFilePath (dirName));
    }

    return true;
}

void FileIndexer::removeFolderFiles (QString folderPath)
{
    QList<QString> words = p.fileWords.keys();

    foreach (QString word, words) {
        QList<FileInfo*> fileInfosForWord = p.fileWords.values (word);

        p.fileWords.remove (word);

        foreach (FileInfo *fileInfo, fileInfosForWord) {
            if (QFileInfo (fileInfo->fileName()).absoluteDir().absolutePath() == folderPath) {
                removeFile (fileInfo);
            } else {
                p.fileWords.insertMulti (word, fileInfo);
            }
        }
    }
}

void FileIndexer::removeFile (FileInfo *fileInfo)
{
//  Q_ASSERT (fileInfo->exists());

    QString fileName = QFileInfo (fileInfo->fileName()).fileName();
    QList<QString> fileNameWords = fileName.toLower().split (QRegExp ("(\\s+|-|_|\\W)+"));

    foreach (QString word, fileNameWords) {
        if (word.isEmpty()) {
            continue;
        }
        removeFileFromWord (fileInfo->fileIndex(), word);
    }

    p.files.remove (fileInfo->fileIndex());
    delete fileInfo;
}

void FileIndexer::removeFileFromWord (uint fileIndex, QString &word)
{
    QList<FileInfo*> fileInfosForWord = p.fileWords.values (word);

    QList<FileInfo*>::iterator it = fileInfosForWord.begin();

    for (;it<fileInfosForWord.end(); it++) {
        if ((*it)->fileIndex() == fileIndex) {
            fileInfosForWord.erase (it);
            break;
        }
    }

    p.fileWords.remove (word);

    if (fileInfosForWord.isEmpty()) {
        //p.queryRoutingFilter->removeFromQueryRoutingTable (word);
    } else {
        foreach (FileInfo *fileInfo, fileInfosForWord) {
            p.fileWords.insertMulti (word, fileInfo);
        }
    }
}

QList<FileInfo*> FileIndexer::match (QString searchCriteria) const
{
    QList<FileInfo*> fileMatches;
    QList<FileInfo*> tempFileInfos;

    QList<QString> searchKeyWords = searchCriteria.split (QRegExp ("\\s+"));

    fileMatches = p.fileWords.values (searchKeyWords.first());
    searchKeyWords.pop_front();

    foreach (QString keyWord, searchKeyWords) {
        tempFileInfos = p.fileWords.values (keyWord);
        fileMatches = (fileMatches.toSet().intersect (tempFileInfos.toSet())).toList();
    }

    return fileMatches;
}

void FileIndexer::CheckIndexThread::run()
{
    fileIndexer->refreshIndex();
}
FileIndexer::CheckIndexThread::CheckIndexThread (FileIndexer *fileIndexer)
 :  fileIndexer (fileIndexer)
{
}

Copyright © 2006 Peter Dimov.
Powered by Doxygen.