当前位置: 首页>>代码示例>>C++>>正文


C++ CSphVector类代码示例

本文整理汇总了C++中CSphVector的典型用法代码示例。如果您正苦于以下问题:C++ CSphVector类的具体用法?C++ CSphVector怎么用?C++ CSphVector使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

在下文中一共展示了CSphVector类的14个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的C++代码示例。

示例1: StrSplit

// copied over from sphinxutils; remove at some point
void StrSplit ( CSphVector<CSphString> & dOut, const char * sIn )
{
	if ( !sIn )
		return;

	const char * p = (char*)sIn;
	while ( *p )
	{
		// skip non-alphas
		while ( (*p) && !IsAlpha(*p) )
			p++;
		if ( !(*p) )
			break;

		// this is my next token
		assert ( IsAlpha(*p) );
		const char * sNext = p;
		while ( IsAlpha(*p) )
			p++;
		if ( sNext!=p )
			dOut.Add().SetBinary ( sNext, p-sNext );
	}
}
开发者ID:baisk,项目名称:csft,代码行数:24,代码来源:wordbreaker.cpp


示例2: strlen

char * ExcerptGen_c::BuildExcerpt ( const ExcerptQuery_t & q, CSphDict * pDict, ISphTokenizer * pTokenizer )
{
	m_dTokens.Reserve ( 1024 );
	m_sBuffer = q.m_sSource;

	const bool bUtf8 = pTokenizer->IsUtf8();
	m_bUtf8 = bUtf8;
	// tokenize query words
	int iWordsLength = strlen ( q.m_sWords.cstr() );

	CSphVector<char> dKwBuffer ( iWordsLength );
	CSphVector<Keyword_t> dKeywords;
	dKeywords.Reserve ( MAX_HIGHLIGHT_WORDS );

	BYTE * sWord;
	int iKwIndex = 0;

	pTokenizer->SetBuffer ( (BYTE*)q.m_sWords.cstr(), iWordsLength );
	while ( ( sWord = pTokenizer->GetToken() ) != NULL )
	{
		SphWordID_t iWord = pDict->GetWordID ( sWord );
		if ( iWord )
		{
			m_dWords.Resize ( m_dWords.GetLength () + 1 );
			Token_t & tLast = m_dWords.Last ();
			tLast.m_eType = TOK_WORD;
			tLast.m_iWordID = iWord;
			tLast.m_iLengthBytes = strlen ( (const char *)sWord );
			tLast.m_iLengthCP = bUtf8 ? sphUTF8Len ( (const char *)sWord ) : tLast.m_iLengthBytes;

			// store keyword
			dKeywords.Resize( dKeywords.GetLength() + 1 );
			Keyword_t & kwLast = dKeywords.Last ();

			// find stars
			bool bStarBack = *pTokenizer->GetTokenEnd() == '*';
			bool bStarFront = ( pTokenizer->GetTokenStart() != pTokenizer->GetBufferPtr() ) &&
				pTokenizer->GetTokenStart()[-1] == '*';
			kwLast.m_uStar = ( bStarFront ? STAR_FRONT : 0 ) | ( bStarBack ? STAR_BACK : 0 );

			// store token
			const int iEndIndex = iKwIndex + tLast.m_iLengthBytes + 1;
			dKwBuffer.Resize ( iEndIndex );
			kwLast.m_iWord = iKwIndex;
			strcpy ( &dKwBuffer [ iKwIndex ], (const char *)sWord );
			iKwIndex = iEndIndex;

			if ( m_dWords.GetLength() == MAX_HIGHLIGHT_WORDS )
				break;
		}
	}

	// tokenize document
	pTokenizer->SetBuffer ( (BYTE*)q.m_sSource.cstr (), strlen ( q.m_sSource.cstr () ) );

	const char * pStartPtr = pTokenizer->GetBufferPtr ();
	const char * pLastTokenEnd = pStartPtr;

	//assign utf-8
	m_sBufferUTF8 = pStartPtr;

	while ( ( sWord = pTokenizer->GetToken() ) != NULL )
	{
		const char * pTokenStart = pTokenizer->GetTokenStart ();

		if ( pTokenStart != pStartPtr )
			AddJunk ( pLastTokenEnd - pStartPtr,
					  pTokenStart - pLastTokenEnd,
					  pTokenizer->GetBoundary() ? pTokenizer->GetBoundaryOffset() : -1 );

		SphWordID_t iWord = pDict->GetWordID ( sWord );

		pLastTokenEnd = pTokenizer->GetTokenEnd ();

		m_dTokens.Resize ( m_dTokens.GetLength () + 1 );
		Token_t & tLast = m_dTokens.Last ();
		tLast.m_eType	= iWord ? TOK_WORD : TOK_SPACE;
		tLast.m_iStart  = pTokenStart - pStartPtr;
		tLast.m_iLengthBytes = pLastTokenEnd - pTokenStart;
		tLast.m_iWordID = iWord;
		tLast.m_uWords = 0;

		// fill word mask
		if ( iWord )
		{
			bool bMatch = false;
			int iOffset;

			ARRAY_FOREACH ( nWord, m_dWords )
			{
				const char * keyword = &dKwBuffer [ dKeywords[nWord].m_iWord ];
				const Token_t & token = m_dWords[nWord];

				switch ( dKeywords[nWord].m_uStar )
				{
				case STAR_NONE:
					bMatch = iWord == token.m_iWordID;
					break;

				case STAR_FRONT:
//.........这里部分代码省略.........
开发者ID:YazaiHu,项目名称:coreseek,代码行数:101,代码来源:sphinxexcerpt.cpp


示例3: TryToExec

bool CSphConfigParser::TryToExec ( char * pBuffer, char * pEnd, const char * szFilename, CSphVector<char> & dResult )
{
	int dPipe[2] = { -1, -1 };

	if ( pipe ( dPipe ) )
	{
		snprintf ( m_sError, sizeof ( m_sError ), "pipe() failed (error=%s)", strerror(errno) );
		return false;
	}

	pBuffer = trim ( pBuffer );

	int iRead  = dPipe [0];
	int iWrite = dPipe [1];

	signal ( SIGCHLD, sigchld );

	int iChild = fork();

	if ( iChild == 0 )
	{
		close ( iRead );
		close ( STDOUT_FILENO );
		dup2 ( iWrite, STDOUT_FILENO );

		char * pPtr = pBuffer;
		char * pArgs = NULL;
		while ( *pPtr )
		{
			if ( sphIsSpace ( *pPtr ) )
			{
				*pPtr = '\0';
				pArgs = trim ( pPtr+1 );
				break;
			}

			pPtr++;
		}
		
		if ( pArgs )
			execl ( pBuffer, pBuffer, pArgs, szFilename, NULL );
		else
			execl ( pBuffer, pBuffer, szFilename, NULL );

		exit ( 1 );
	}
	else
		if ( iChild == -1 )
		{
			snprintf ( m_sError, sizeof ( m_sError ), "fork failed (error=%s)", strerror(errno) );
			return false;
		}

	close ( iWrite );

	int iBytesRead, iTotalRead = 0;
	const int BUFFER_SIZE = 65536;
	
	dResult.Reset ();

	do
	{
		dResult.Resize ( iTotalRead + BUFFER_SIZE );
		iBytesRead = read ( iRead, (void*)&(dResult [iTotalRead]), BUFFER_SIZE );
		iTotalRead += iBytesRead;
	}
	while ( iBytesRead > 0 );

	int iStatus;
	wait ( &iStatus );
	iStatus = (signed char) WEXITSTATUS (iStatus);

	if ( iStatus )
	{
		snprintf ( m_sError, sizeof ( m_sError ), "error executing '%s'", pBuffer );
		return false;
	}

	if ( iBytesRead < 0  )
	{
		snprintf ( m_sError, sizeof ( m_sError ), "pipe read error (error=%s)", strerror(errno) );
		return false;
	}

	dResult.Resize ( iTotalRead + 1 );
	dResult [iTotalRead] = '\0';

	return true;
}
开发者ID:agibralter,项目名称:sphinx,代码行数:89,代码来源:sphinxutils.cpp


示例4: UrlBreak

void UrlBreak ( Split_t & tBest, const char * sWord )
{
	const int iLen = strlen(sWord);

	tBest.m_Pos.Resize(0);

	// current partial splits
	// begin with an empty one
	CSphVector<Split_t> dSplits;
	dSplits.Add();

	// our best guess so far
	// begin with a trivial baseline one (ie. no splits at all)
	Prob_t p = g_LM.GetProb ( sWord, iLen );
	tBest.m_Pos.Add ( iLen );
	tBest.m_fProb = p.m_fProb;
	tBest.m_bAllDict = tBest.m_bAnyDict = p.m_bDict;

	if ( iLen>=DICT_COMPOUND_MIN && tBest.m_bAllDict )
	{
		static const float THRESH = logf ( DICT_COMPOUND_THRESH );
		if ( tBest.m_fProb<=THRESH )
			tBest.m_fProb *= DICT_COMPOUND_COEFF;
	}

	// work the current splits
	CSphVector<Split_t> dSplits2;
	while ( dSplits.GetLength() )
	{
		int iWorkedSplits = 0;
		float fPrevBest = tBest.m_fProb;

		ARRAY_FOREACH ( iSplit, dSplits )
		{
			Split_t & s = dSplits[iSplit];

			// filter out splits that were added before (!) a new best guess on the previous iteration
			if ( dSplits[iSplit] < tBest )
				continue;
			iWorkedSplits++;

			int iLast = 0;
			if ( s.m_Pos.GetLength() )
				iLast = s.m_Pos.Last();

			for ( int i=1+iLast; i<iLen; i++ )
			{
				// consider a split at position i
				// it generates a word candidate [iLast,i) and a tail [i,iLen)
				// let's score those
				Prob_t tCand = g_LM.GetProb ( sWord+iLast, i-iLast );
				Prob_t tTail = g_LM.GetProb ( sWord+i, iLen-i );

				// if the current best is all-keywords, the new candidates must be, too
				if ( tBest.m_bAllDict && !tCand.m_bDict )
					continue;

				// compute partial and full split candidates generated by the current guess
				Split_t tPartial = s;
				tPartial.AddSplitPos ( tCand, i );

				Split_t tFull = tPartial;
				tFull.AddSplitPos ( tTail, iLen );

				// check if the full one is our new best full one
				bool bNewBest = false;
				if ( tBest < tFull )
				{
					// FIXME? we do this even when the new split is *not* all-keywords,
					// but the old best split was; is this ever a problem?
					tBest = tFull;
//					tBest.Dump ( sWord, "new-best" );
					bNewBest = true;
				}

				// check if the resulting partial split is worth scanning further
				if ( tBest < tPartial )
				{
					dSplits2.Add ( tPartial );
//					dSplits2.Last().Dump ( sWord, "scan-partial" );
				}
			}
		}

		// damage control!
		// if we just processed over 100K candidate splits and got no improvement
		// lets assume that our chances of getting one are kinda low and bail
		if ( iWorkedSplits>=100000 && tBest.m_fProb>=fPrevBest )
			break;

		// keep going
		dSplits.SwapData ( dSplits2 );
		dSplits2.Resize ( 0 );
	}
开发者ID:baisk,项目名称:csft,代码行数:94,代码来源:wordbreaker.cpp


示例5: main


//.........这里部分代码省略.........
		if ( pWord->m_sFlags.IsEmpty() )
			continue;

		CSphString sWord, sWordForCross;
		int iFlagLen = strlen ( pWord->m_sFlags.cstr () );
		for ( int iFlag1 = 0; iFlag1 < iFlagLen; ++iFlag1 )
			for ( int iRule1 = 0; iRule1 < Affix.GetNumRules (); ++iRule1 )
			{
				CISpellAffixRule * pRule1 = Affix.GetRule ( iRule1 );
				if ( pRule1->Flag()!=pWord->m_sFlags.cstr()[iFlag1] )
					continue;

				sWord = pWord->m_sWord;

				if ( !pRule1->Apply ( sWord ) )
					continue;

				EmitResult ( tWordMap, sWord, pWord->m_sWord, pRule1->Flag() );

				// apply other rules
				if ( !Affix.CheckCrosses() )
					continue;

				if ( !pRule1->IsCrossProduct() )
					continue;

				for ( int iFlag2 = iFlag1 + 1; iFlag2 < iFlagLen; ++iFlag2 )
					for ( int iRule2 = 0; iRule2 < Affix.GetNumRules (); ++iRule2 )
					{
						CISpellAffixRule * pRule2 = Affix.GetRule ( iRule2 );
						if ( !pRule2->IsCrossProduct () || pRule2->Flag()!=pWord->m_sFlags.cstr()[iFlag2] ||
							pRule2->IsPrefix()==pRule1->IsPrefix() )
							continue;

						sWordForCross = sWord;
						if ( pRule2->Apply ( sWordForCross ) )
							EmitResult ( tWordMap, sWordForCross, pWord->m_sWord, pRule1->Flag(), pRule2->Flag() );
					}
			}
	}
	printf ( "\rDictionary words processed: %d\n", nDone );

	// output

	CSphVector<const char *> dKeys;
	tWordMap.IterateStart();
	while ( tWordMap.IterateNext() )
		dKeys.Add ( tWordMap.IterateGetKey().cstr() );
	dKeys.Sort ( WordLess() );

	ARRAY_FOREACH ( iKey, dKeys )
	{
		const CSphVector<MapInfo_t> & dWords = tWordMap[dKeys[iKey]];
		const char * sKey = dKeys[iKey];

		switch ( eMode )
		{
			case M_LAST:
				fprintf ( pFile, "%s > %s\n", sKey, dWords.Last().m_sWord.cstr() );
				break;

			case M_EXACT_OR_LONGEST:
			{
				int iMatch = 0;
				int iLength = 0;

				ARRAY_FOREACH ( i, dWords )
				{
					if ( dWords[i].m_sWord==sKey )
					{
						iMatch = i;
						break;
					}

					int iWordLength = strlen ( dWords[i].m_sWord.cstr() );
					if ( iWordLength>iLength )
					{
						iLength = iWordLength;
						iMatch = i;
					}
				}

				fprintf ( pFile, "%s > %s\n", sKey, dWords[iMatch].m_sWord.cstr() );
				break;
			}

			case M_DUPLICATES:
				if ( dWords.GetLength()==1 ) break;
			case M_DEBUG:
				ARRAY_FOREACH ( i, dWords )
					fprintf ( pFile, "%s > %s %s/%d\n", sKey, dWords[i].m_sWord.cstr(),
						dWords[i].m_sRules, dWords.GetLength() );
				break;
		}
	}

	fclose ( pFile );

	return 0;
}
开发者ID:dandanlee,项目名称:coreseek-1,代码行数:101,代码来源:spelldump.cpp


示例6: sphPluginReload

bool sphPluginReload ( const char * sName, CSphString & sError )
{
#if !HAVE_DLOPEN
	sError = "no dlopen(), no plugins";
	return false;
#else
	// find all plugins from the given library
	CSphScopedLock<CSphMutex> tLock ( g_tPluginMutex );

	CSphVector<PluginKey_t> dKeys;
	CSphVector<PluginDesc_c*> dPlugins;

	g_hPlugins.IterateStart();
	while ( g_hPlugins.IterateNext() )
	{
		PluginDesc_c * v = g_hPlugins.IterateGet();
		if ( v->GetLibName()==sName )
		{
			dKeys.Add ( g_hPlugins.IterateGetKey() );
			dPlugins.Add ( g_hPlugins.IterateGet() );
		}
	}

	// no plugins loaded? oops
	if ( dPlugins.GetLength()==0 )
	{
		sError.SetSprintf ( "no active plugins loaded from %s", sName );
		return false;
	}

	// load new library and check every plugin
#if !USE_WINDOWS
	PluginLib_c * pNewLib = LoadPluginLibrary ( sName, sError, true );
#else
	PluginLib_c * pNewLib = LoadPluginLibrary ( sName, sError );
#endif
	if ( !pNewLib )
		return false;

	// load all plugins
	CSphVector<PluginDesc_c*> dNewPlugins;
	ARRAY_FOREACH ( i, dPlugins )
	{
		PluginDesc_c * pDesc = NULL;
		const SymbolDesc_t * pSym = NULL;
		switch ( dKeys[i].m_eType )
		{
			case PLUGIN_RANKER:					pDesc = new PluginRanker_c ( pNewLib ); pSym = g_dSymbolsRanker; break;
			case PLUGIN_INDEX_TOKEN_FILTER:		pDesc = new PluginTokenFilter_c ( pNewLib ); pSym = g_dSymbolsTokenFilter; break;
			case PLUGIN_QUERY_TOKEN_FILTER:		pDesc = new PluginQueryTokenFilter_c ( pNewLib ); pSym = g_dSymbolsQueryTokenFilter; break;
			case PLUGIN_FUNCTION:				pDesc = new PluginUDF_c ( pNewLib, dPlugins[i]->GetUdfRetType() ); pSym = g_dSymbolsUDF; break;
			default:
				sphDie ( "INTERNAL ERROR: unknown plugin type %d in sphPluginReload()", (int)dKeys[i].m_eType );
				return false;
		}

		if ( !PluginLoadSymbols ( pDesc, pSym, pNewLib->GetHandle(), dKeys[i].m_sName.cstr(), sError ) )
		{
			pDesc->Release();
			break;
		}

		dNewPlugins.Add ( pDesc );
	}
开发者ID:1m69,项目名称:sphinx,代码行数:64,代码来源:sphinxplugin.cpp


示例7: AddBoundary

void ExcerptGen_c::AddBoundary()
{
	Token_t & tLast = m_dTokens.Add();
	tLast.m_eType = TOK_BREAK;
	tLast.m_iStart = 0;
	tLast.m_iLengthBytes = 0;
	tLast.m_iWordID = 0;
	tLast.m_uWords = 0;
	tLast.m_uPosition = 0;
}
开发者ID:jzawodn,项目名称:Sphinx-at-Craigslist,代码行数:10,代码来源:sphinxexcerpt.cpp


示例8: AddJunk

void ExcerptGen_c::AddJunk ( int iStart, int iLength, int iBoundary )
{
	assert ( iLength>0 );
	assert ( iLength<=m_sBuffer.Length() );
	assert ( iStart+iLength<=m_sBuffer.Length() );

	int iChunkStart = iStart;
	int iSaved = 0;

	for ( int i = iStart; i < iStart+iLength; i++ )
		if ( sphIsSpace ( m_sBuffer.cstr () [i] )!=sphIsSpace ( m_sBuffer.cstr () [iChunkStart] ) )
		{
			Token_t & tLast = m_dTokens.Add();
			tLast.m_eType = TOK_SPACE;
			tLast.m_iStart = iChunkStart;
			tLast.m_iLengthBytes = i - iChunkStart;
			tLast.m_iWordID = 0;
			tLast.m_uWords = 0;
			tLast.m_uPosition = 0;

			iChunkStart = i;
			iSaved += tLast.m_iLengthBytes;

			if ( iBoundary!=-1 && iSaved > ( iBoundary-iStart ) )
			{
				AddBoundary();
				iBoundary = -1;
			}
		}

	Token_t & tLast = m_dTokens.Add();
	tLast.m_eType = TOK_SPACE;
	tLast.m_iStart = iChunkStart;
	tLast.m_iLengthBytes = iStart + iLength - iChunkStart;
	tLast.m_iWordID = 0;
	tLast.m_uWords = 0;
	tLast.m_uPosition = 0;

	if ( iBoundary!=-1 )
		AddBoundary();
}
开发者ID:jzawodn,项目名称:Sphinx-at-Craigslist,代码行数:41,代码来源:sphinxexcerpt.cpp


示例9: AddJunk

void ExcerptGen_c::AddJunk ( int iStart, int iLength )
{
    int iChunkStart = iStart;

    for ( int i = iStart; i < iStart+iLength; i++ )
        if ( sphIsSpace ( m_sBuffer.cstr () [i] ) != sphIsSpace ( m_sBuffer.cstr () [iChunkStart] ) )
        {
            m_dTokens.Resize ( m_dTokens.GetLength () + 1 );
            Token_t & tLast = m_dTokens.Last ();
            tLast.m_eType   = TOK_SPACE;
            tLast.m_iStart	= iChunkStart;
            tLast.m_iLengthBytes = i - iChunkStart;
            tLast.m_iWordID = 0;
            tLast.m_uWords = 0;

            iChunkStart = i;
        }

    m_dTokens.Resize ( m_dTokens.GetLength () + 1 );
    Token_t & tLast = m_dTokens.Last ();
    tLast.m_eType   = TOK_SPACE;
    tLast.m_iStart	= iChunkStart;
    tLast.m_iLengthBytes = iStart + iLength - iChunkStart;
    tLast.m_iWordID = 0;
    tLast.m_uWords = 0;
}
开发者ID:aughey,项目名称:sphinx-sqlite3,代码行数:26,代码来源:sphinxexcerpt.cpp


示例10: AddJunk

void ExcerptGen_c::AddJunk ( int iStart, int iLength, int iBoundary )
{
	int iChunkStart = iStart;
	int iSaved = 0;

	for ( int i = iStart; i < iStart+iLength; i++ ){
		const char* buf_ptr = NULL;
		if(m_bUtf8){
			buf_ptr = m_sBufferUTF8.cstr ();
		}else{
			buf_ptr = m_sBuffer.cstr ();
		}
		if ( sphIsSpace (  buf_ptr[i] ) != sphIsSpace ( buf_ptr[iChunkStart] ) )
		{
			m_dTokens.Resize ( m_dTokens.GetLength () + 1 );
			Token_t & tLast = m_dTokens.Last ();
			tLast.m_eType   = TOK_SPACE;
			tLast.m_iStart	= iChunkStart;
			tLast.m_iLengthBytes = i - iChunkStart;
			tLast.m_iWordID = 0;
			tLast.m_uWords = 0;

			iChunkStart = i;
			iSaved += tLast.m_iLengthBytes;

			if ( iBoundary != -1 && iSaved > iBoundary - iStart )
			{
				AddBoundary();
				iBoundary = -1;
			}
		}
	}

	m_dTokens.Resize ( m_dTokens.GetLength () + 1 );
	Token_t & tLast = m_dTokens.Last ();
	tLast.m_eType   = TOK_SPACE;
	tLast.m_iStart	= iChunkStart;
	tLast.m_iLengthBytes = iStart + iLength - iChunkStart;
	tLast.m_iWordID = 0;
	tLast.m_uWords = 0;

	if ( iBoundary != -1 ) AddBoundary();
}
开发者ID:YazaiHu,项目名称:coreseek,代码行数:43,代码来源:sphinxexcerpt.cpp


示例11: GetNumRules

int CISpellAffix::GetNumRules () const
{
	return m_dRules.GetLength ();
}
开发者ID:dandanlee,项目名称:coreseek-1,代码行数:4,代码来源:spelldump.cpp


示例12: LoadMySpell

bool CISpellAffix::LoadMySpell ( FILE * pFile )
{
	char sBuffer	[MAX_STR_LENGTH];
	char sCondition	[MAX_STR_LENGTH];
	char sRemove	[MAX_STR_LENGTH];
	char sAppend	[MAX_STR_LENGTH];

	RuleType_e eRule = RULE_NONE;
	BYTE cFlag = 0;
	BYTE cCombine = 0;
	int iCount = 0, iLine = 0;
	const char * sMode = 0;

	while ( !feof ( pFile ) )
	{
		char * sLine = fgets ( sBuffer, MAX_STR_LENGTH, pFile );
		if ( !sLine )
			break;
		++iLine;

		// prefix and suffix rules
		RuleType_e eNewRule = RULE_NONE;
		if ( !strncmp ( sLine, "PFX", 3 ) )
		{
			eNewRule = RULE_PREFIXES;
			sMode = "prefix";

		} else if ( !strncmp ( sLine, "SFX", 3 ) )
		{
			eNewRule = RULE_SUFFIXES;
			sMode = "suffix";
		}

		if ( eNewRule!=RULE_NONE )
		{
			sLine += 3;
			while ( *sLine && isspace ( (unsigned char) *sLine ) )
				++sLine;

			if ( eNewRule!=eRule ) // new rule header
			{
				if ( iCount )
					printf ( "WARNING: Line %d: Premature end of entries.\n", iLine );

				if ( sscanf ( sLine, "%c %c %d", &cFlag, &cCombine, &iCount )!=3 ) // NOLINT
					printf ( "WARNING; Line %d: Malformed %s header\n", iLine, sMode );

				eRule = eNewRule;

			} else // current rule continued
			{
				*sRemove = *sAppend = 0;
				char cNewFlag;
				if ( sscanf ( sLine, "%c %s %s %s", &cNewFlag, sRemove, sAppend, sCondition )==4 ) // NOLINT
				{
					if ( cNewFlag!=cFlag )
						printf ( "WARNING: Line %d: Flag character mismatch\n", iLine );

					if ( *sRemove=='0' && *(sRemove + 1)==0 ) *sRemove = 0;
					if ( *sAppend=='0' && *(sAppend + 1)==0 ) *sAppend = 0;

					CISpellAffixRule Rule ( eRule, cFlag, cCombine=='Y', sCondition, sRemove, sAppend );
					m_dRules.Add ( Rule );

				} else
					printf ( "WARNING: Line %d: Malformed %s rule\n", iLine, sMode );

				if ( !--iCount ) eRule = RULE_NONE;
			}
			continue;
		}
	}
	return true;
}
开发者ID:dandanlee,项目名称:coreseek-1,代码行数:74,代码来源:spelldump.cpp


示例13: LoadISpell

bool CISpellAffix::LoadISpell ( FILE * pFile )
{
	char szBuffer [ MAX_STR_LENGTH ];
	char szCondition [ MAX_STR_LENGTH ];
	char szStrip [ MAX_STR_LENGTH ];
	char szAppend [ MAX_STR_LENGTH ];

	RuleType_e eRule = RULE_NONE;
	char cFlag = '\0';
	bool bCrossProduct = false;
	int iLine = 0;

	// TODO: parse all .aff character replacement commands
	while ( !feof ( pFile ) )
	{
		char * szResult = fgets ( szBuffer, MAX_STR_LENGTH, pFile );
		if ( !szResult )
			break;
		iLine++;

		if ( !strncasecmp ( szBuffer, "prefixes", 8 ) )
		{
			eRule = RULE_PREFIXES;
			continue;
		}

		if ( !strncasecmp ( szBuffer, "suffixes", 8 ) )
		{
			eRule = RULE_SUFFIXES;
			continue;
		}

		if ( !strncasecmp ( szBuffer, "wordchars", 9 ) )
		{
			char * szStart = szBuffer + 9;
			while ( *szStart && isspace ( (unsigned char) *szStart ) )
				++szStart;

			char * szRangeL = szStart;
			while ( *szStart && !isspace ( (unsigned char) *szStart ) )
				++szStart;

			if ( !*szStart )
			{
				printf ( "WARNING: Line %d: invalid 'wordchars' statement\n", iLine );
				continue;
			}

			*szStart = '\0';
			++szStart;

			while ( *szStart && isspace ( (unsigned char) *szStart ) )
				++szStart;

			char * szRangeU = szStart;

			while ( *szStart && !isspace ( (unsigned char) *szStart ) )
				++szStart;

			*szStart = '\0';

			if ( !AddToCharset ( szRangeL, szRangeU ) )
				printf ( "WARNING: Line %d: cannot add to charset: '%s' '%s'\n", iLine, szRangeL, szRangeU );

			continue;
		}

		if ( !strncasecmp ( szBuffer, "flag", 4 ) )
		{
			if ( eRule==RULE_NONE )
			{
				printf ( "WARNING: Line %d: 'flag' appears before preffixes or suffixes\n", iLine );
				continue;
			}

			char * szStart = szBuffer + 4;
			while ( *szStart && isspace ( (unsigned char) *szStart ) )
				++szStart;

			bCrossProduct = ( *szStart=='*' );

			cFlag = bCrossProduct ? *(szStart + 1) : *(szStart);
			continue;
		}

		if ( eRule==RULE_NONE )
			continue;

		char * szComment = strchr ( szBuffer, '#' );
		if ( szComment )
			*szComment = '\0';

		if ( !* szBuffer )
			continue;

		szCondition[0] = '\0';
		szStrip[0] = '\0';
		szAppend[0] = '\0';

		int nFields = sscanf ( szBuffer, "%[^>\n]>%[^,\n],%[^\n]", szCondition, szStrip, szAppend ); // NOLINT
//.........这里部分代码省略.........
开发者ID:dandanlee,项目名称:coreseek-1,代码行数:101,代码来源:spelldump.cpp


示例14: TokenizeDocument

void ExcerptGen_c::TokenizeDocument ( char * pData, CSphDict * pDict, ISphTokenizer * pTokenizer, bool bFillMasks, bool bRetainHtml )
{
	m_iDocumentWords = 0;
	m_dTokens.Reserve ( 1024 );
	m_sBuffer = pData;

	pTokenizer->SetBuffer ( (BYTE*)pData, strlen(pData) );

	const char * pStartPtr = pTokenizer->GetBufferPtr ();
	const char * pLastTokenEnd = pStartPtr;

	if ( bRetainHtml )
		pTokenizer->AddSpecials ( "<" );

	BYTE * sWord;
	DWORD uPosition = 0; // hit position in document
	while ( ( sWord = pTokenizer->GetToken() )!=NULL )
	{
		if ( pTokenizer->TokenIsBlended() )
			continue;

		const char * pTokenStart = pTokenizer->GetTokenStart ();

		if ( pTokenStart!=pStartPtr && pTokenStart>pLastTokenEnd )
			AddJunk ( pLastTokenEnd - pStartPtr,
				pTokenStart - pLastTokenEnd,
				pTokenizer->GetBoundary() ? pTokenizer->GetBoundaryOffset() : -1 );

		if ( bRetainHtml && *pTokenStart=='<' )
		{
			int iTagEnd = FindTagEnd ( pTokenStart );
			if ( iTagEnd!=-1 )
			{
				assert ( pTokenStart+iTagEnd<pTokenizer->GetBufferEnd() );
				AddJunk ( pTokenStart-pStartPtr, iTagEnd+1, pTokenizer->GetBoundary() ? pTokenizer->GetBoundaryOffset() : -1 );
				pTokenizer->SetBufferPtr ( pTokenStart+iTagEnd+1 );
				pLastTokenEnd = pTokenStart+iTagEnd+1; // fix it up to prevent adding last chunk on exit
				continue;
			}
		}

		SphWordID_t iWord = iWord = pDict->GetWordID ( sWord );

		pLastTokenEnd = pTokenizer->GetTokenEnd ();

		if ( pTokenizer->GetBoundary() )
			uPosition += 100; // FIXME: this should be taken from index settings

		Token_t & tLast = m_dTokens.Add();
		tLast.m_eType = iWord ? TOK_WORD : TOK_SPACE;
		tLast.m_uPosition = iWord ? ++uPosition : 0;
		tLast.m_iStart = pTokenStart - pStartPtr;
		tLast.m_iLengthBytes = pLastTokenEnd - pTokenStart;
		tLast.m_iWordID = iWord;
		tLast.m_uWords = 0;
		if ( iWord )
			m_iDocumentWords++;

		m_iLastWord = iWord ? m_dTokens.GetLength() - 1 : m_iLastWord;

		// fill word mask
		if ( bFillMasks && iWord )
		{
			bool bMatch = false;
			int iOffset;

			ARRAY_FOREACH ( nWord, m_dWords )
			{
				const char * sKeyword = &m_dKeywordsBuffer [ m_dKeywords[nWord].m_iWord ];
				const Token_t & tToken = m_dWords[nWord];

				switch ( m_dKeywords[nWord].m_uStar )
				{
				case STAR_NONE:
					bMatch = ( iWord==tToken.m_iWordID );
					break;

				case STAR_FRONT:
					iOffset = tLast.m_iLengthBytes - tToken.m_iLengthBytes;
					bMatch = ( iOffset>=0 ) &&
						( memcmp ( sKeyword, sWord + iOffset, tToken.m_iLengthBytes )==0 );
					break;

				case STAR_BACK:
					bMatch = ( tLast.m_iLengthBytes>=tToken.m_iLengthBytes ) &&
						( memcmp ( sKeyword, sWord, tToken.m_iLengthBytes )==0 );
					break;

				case STAR_BOTH:
					bMatch = strstr ( (const char *)sWord, sKeyword )!=NULL;
					break;
				}

				if ( bMatch )
				{
					tLast.m_uWords |= 1UL<<nWord;
					m_uFoundWords |= 1UL<<nWord;
				}
			}
		}
//.........这里部分代码省略.........
开发者ID:jzawodn,项目名称:Sphinx-at-Craigslist,代码行数:101,代码来源:sphinxexcerpt.cpp



注:本文中的CSphVector类示例整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。