How can I split a string by a delimiter into an array?

You can use a vector of string (std::vector<std::string>), append each token to it with push_back, and then return it from your tokenize function.

Here's my first attempt at this using vectors and strings:

vector<string> explode(const string& str, const char& ch) {
    string next;
    vector<string> result;

    // For each character in the string
    for (string::const_iterator it = str.begin(); it != str.end(); it++) {
        // If we've hit the terminal character
        if (*it == ch) {
            // If we have some characters accumulated
            if (!next.empty()) {
                // Add them to the result vector
                result.push_back(next);
                next.clear();
            }
        } else {
            // Accumulate the next character into the sequence
            next += *it;
        }
    }
    if (!next.empty())
         result.push_back(next);
    return result;
}

Hopefully this gives you some sort of idea of how to go about this. On your example string it returns the correct results with this test code:

int main (int, char const **) {
    std::string blah = "___this_ is__ th_e str__ing we__ will use__";
    std::vector<std::string> result = explode(blah, '_');

    for (size_t i = 0; i < result.size(); i++) {
        cout << "\"" << result[i] << "\"" << endl;
    }
    return 0;
}

Using STL (sorry no compiler not tested)

#include <vector>
#include <string>
#include <sstream>

int main()
{
    std::vector<std::string>   result;

    std::string str = "___this_ is__ th_e str__ing we__ will use__";

    std::stringstream  data(str);

    std::string line;
    while(std::getline(data,line,'_'))
    {
        result.push_back(line); // Note: You may get a couple of blank lines
                                // When multiple underscores are beside each other.
    }
}

// or define a token

#include <vector>
#include <string>
#include <iterator>
#include <algorithm>
#include <sstream>

struct Token: public std::string  // Yes I know this is nasty.
{                                 // But it is just to demosntrate the principle.    
};

std::istream& operator>>(std::istream& s,Token& t)
{
    std::getline(s,t,'_');
   
    // *** 
    // Remove extra '_' characters from the stream.
    char c;
    while(s && ((c = s.get()) != '_')) {/*Do Nothing*/}
    if (s)
    {
        s.unget(); // Put back the last char as it is not '_'
    }
    return s;
}

int main()
{   

    std::string str = "___this_ is__ th_e str__ing we__ will use__";

    std::stringstream  data(str);

    std::vector<std::string>   result(std::istream_iterator<Token>(data),
                                      std::istream_iterator<Token>());
}

It works for me :

#include <iostream>
#include <vector>
#include <string>

using namespace std;

vector<string> explode( const string &delimiter, const string &explodeme);

int main(int argc, char *argv[])
{
    string str = "I have a lovely bunch of cocoa nuts";
    cout<<str<<endl;
    vector<string> v = explode(" ", str);
    for(int i=0; i<v.size(); i++)
        cout <<i << " ["<< v[i] <<"] " <<endl;
}

vector<string> explode( const string &delimiter, const string &str)
{
    vector<string> arr;

    int strleng = str.length();
    int delleng = delimiter.length();
    if (delleng==0)
        return arr;//no change

    int i=0;
    int k=0;
    while( i<strleng )
    {
        int j=0;
        while (i+j<strleng && j<delleng && str[i+j]==delimiter[j])
            j++;
        if (j==delleng)//found delimiter
        {
            arr.push_back(  str.substr(k, i-k) );
            i+=delleng;
            k=i;
        }
        else
        {
            i++;
        }
    }
    arr.push_back(  str.substr(k, i-k) );
    return arr;
}

source : http://www.zedwood.com/article/106/cpp-explode-function